2024-11-09 00:21:23,839 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 00:21:23,860 main DEBUG Took 0.017452 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 00:21:23,861 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 00:21:23,862 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 00:21:23,864 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 00:21:23,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,886 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 00:21:23,902 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,905 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,909 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,911 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,913 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,916 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,917 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,920 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,922 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 00:21:23,923 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,923 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 00:21:23,925 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 00:21:23,927 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 00:21:23,930 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 00:21:23,931 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 00:21:23,933 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 00:21:23,934 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 00:21:23,946 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 00:21:23,949 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 00:21:23,952 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 00:21:23,953 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 00:21:23,953 main DEBUG createAppenders(={Console}) 2024-11-09 00:21:23,954 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 00:21:23,955 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 00:21:23,955 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 00:21:23,956 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 00:21:23,956 main DEBUG OutputStream closed 2024-11-09 00:21:23,957 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 00:21:23,957 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 00:21:23,957 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 00:21:24,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 00:21:24,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 00:21:24,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 00:21:24,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 00:21:24,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 00:21:24,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 00:21:24,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 00:21:24,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 00:21:24,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 00:21:24,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 00:21:24,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 00:21:24,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 00:21:24,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 00:21:24,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 00:21:24,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 00:21:24,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 00:21:24,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 00:21:24,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 00:21:24,103 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 00:21:24,104 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 00:21:24,104 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 00:21:24,105 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T00:21:24,593 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68 2024-11-09 00:21:24,596 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 00:21:24,596 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T00:21:24,606 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-09T00:21:24,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T00:21:24,666 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0, deleteOnExit=true 2024-11-09T00:21:24,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T00:21:24,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/test.cache.data in system properties and HBase conf 2024-11-09T00:21:24,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T00:21:24,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.log.dir in system properties and HBase conf 2024-11-09T00:21:24,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T00:21:24,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T00:21:24,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T00:21:24,826 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T00:21:24,979 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T00:21:24,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T00:21:24,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T00:21:24,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T00:21:24,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T00:21:24,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T00:21:24,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T00:21:24,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T00:21:24,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T00:21:24,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T00:21:24,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/nfs.dump.dir in system properties and HBase conf 2024-11-09T00:21:24,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/java.io.tmpdir in system properties and HBase conf 2024-11-09T00:21:24,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T00:21:24,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T00:21:24,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T00:21:26,378 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T00:21:26,481 INFO [Time-limited test {}] log.Log(170): Logging initialized @3673ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T00:21:26,584 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T00:21:26,659 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T00:21:26,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T00:21:26,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T00:21:26,706 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T00:21:26,725 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T00:21:26,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.log.dir/,AVAILABLE} 2024-11-09T00:21:26,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T00:21:26,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/java.io.tmpdir/jetty-localhost-37817-hadoop-hdfs-3_4_1-tests_jar-_-any-17121712089332892194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T00:21:26,996 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37817} 2024-11-09T00:21:26,996 INFO [Time-limited test {}] server.Server(415): Started @4189ms 2024-11-09T00:21:27,811 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T00:21:27,820 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T00:21:27,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T00:21:27,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T00:21:27,829 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T00:21:27,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.log.dir/,AVAILABLE} 2024-11-09T00:21:27,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T00:21:27,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/java.io.tmpdir/jetty-localhost-36505-hadoop-hdfs-3_4_1-tests_jar-_-any-7189331973034760836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T00:21:27,956 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:36505} 2024-11-09T00:21:27,956 INFO [Time-limited test {}] server.Server(415): Started @5149ms 2024-11-09T00:21:28,022 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T00:21:29,942 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data1/current/BP-1500709718-172.17.0.3-1731111685778/current, will proceed with Du for space computation calculation, 2024-11-09T00:21:29,942 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data2/current/BP-1500709718-172.17.0.3-1731111685778/current, will proceed with Du for space computation calculation, 2024-11-09T00:21:30,033 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T00:21:30,123 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68 2024-11-09T00:21:30,126 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x186f8b7fc1f531f7 with lease ID 0x3c193f5aa149e56c: Processing first storage report for DS-832d4b32-28cf-446d-a540-1a19865fc9da from datanode DatanodeRegistration(127.0.0.1:39715, datanodeUuid=85de3c15-71d5-4162-8dbc-9224020473f9, infoPort=36429, infoSecurePort=0, ipcPort=33323, storageInfo=lv=-57;cid=testClusterID;nsid=786087319;c=1731111685778) 2024-11-09T00:21:30,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x186f8b7fc1f531f7 with lease ID 0x3c193f5aa149e56c: from storage DS-832d4b32-28cf-446d-a540-1a19865fc9da node DatanodeRegistration(127.0.0.1:39715, datanodeUuid=85de3c15-71d5-4162-8dbc-9224020473f9, infoPort=36429, infoSecurePort=0, ipcPort=33323, storageInfo=lv=-57;cid=testClusterID;nsid=786087319;c=1731111685778), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-09T00:21:30,128 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x186f8b7fc1f531f7 with lease ID 0x3c193f5aa149e56c: Processing first storage report for DS-9e9a7cfe-b108-41cc-a1ed-f77d3b2cd3e6 from datanode DatanodeRegistration(127.0.0.1:39715, datanodeUuid=85de3c15-71d5-4162-8dbc-9224020473f9, infoPort=36429, infoSecurePort=0, ipcPort=33323, storageInfo=lv=-57;cid=testClusterID;nsid=786087319;c=1731111685778) 2024-11-09T00:21:30,128 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x186f8b7fc1f531f7 with lease ID 0x3c193f5aa149e56c: from storage DS-9e9a7cfe-b108-41cc-a1ed-f77d3b2cd3e6 node DatanodeRegistration(127.0.0.1:39715, datanodeUuid=85de3c15-71d5-4162-8dbc-9224020473f9, infoPort=36429, infoSecurePort=0, ipcPort=33323, storageInfo=lv=-57;cid=testClusterID;nsid=786087319;c=1731111685778), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T00:21:30,266 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/zookeeper_0, clientPort=60455, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T00:21:30,279 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60455 2024-11-09T00:21:30,295 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:30,300 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:30,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741825_1001 (size=7) 2024-11-09T00:21:31,166 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b with version=8 2024-11-09T00:21:31,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/hbase-staging 2024-11-09T00:21:31,325 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T00:21:31,684 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef986b38239e:0 server-side Connection retries=45 2024-11-09T00:21:31,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:31,716 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:31,723 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T00:21:31,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:31,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T00:21:31,997 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T00:21:32,090 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T00:21:32,104 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T00:21:32,111 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T00:21:32,158 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30796 (auto-detected) 2024-11-09T00:21:32,159 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T00:21:32,187 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38689 2024-11-09T00:21:32,217 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38689 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-09T00:21:32,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386890x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T00:21:32,342 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38689-0x1011cf81b9d0000 connected 2024-11-09T00:21:32,431 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:32,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:32,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T00:21:32,455 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b, hbase.cluster.distributed=false 2024-11-09T00:21:32,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T00:21:32,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38689 2024-11-09T00:21:32,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38689 2024-11-09T00:21:32,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38689 2024-11-09T00:21:32,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38689 2024-11-09T00:21:32,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38689 2024-11-09T00:21:32,648 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef986b38239e:0 server-side Connection retries=45 2024-11-09T00:21:32,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:32,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:32,651 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T00:21:32,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T00:21:32,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T00:21:32,654 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T00:21:32,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T00:21:32,661 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35861 2024-11-09T00:21:32,664 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35861 connecting to ZooKeeper ensemble=127.0.0.1:60455 2024-11-09T00:21:32,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:32,671 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:32,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358610x0, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T00:21:32,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358610x0, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T00:21:32,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35861-0x1011cf81b9d0001 connected 2024-11-09T00:21:32,708 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T00:21:32,718 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T00:21:32,721 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T00:21:32,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T00:21:32,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35861 2024-11-09T00:21:32,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35861 2024-11-09T00:21:32,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35861 2024-11-09T00:21:32,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35861 2024-11-09T00:21:32,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35861 2024-11-09T00:21:32,762 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef986b38239e:38689 2024-11-09T00:21:32,763 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef986b38239e,38689,1731111691424 2024-11-09T00:21:32,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T00:21:32,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T00:21:32,784 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef986b38239e,38689,1731111691424 2024-11-09T00:21:32,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T00:21:32,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:32,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:32,820 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T00:21:32,822 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef986b38239e,38689,1731111691424 from backup master directory 2024-11-09T00:21:32,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef986b38239e,38689,1731111691424 2024-11-09T00:21:32,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T00:21:32,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T00:21:32,830 WARN [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T00:21:32,830 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef986b38239e,38689,1731111691424 2024-11-09T00:21:32,834 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T00:21:32,835 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T00:21:32,915 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/hbase.id] with ID: b39df24e-75e2-4c08-8f69-2ef0b1a146a3 2024-11-09T00:21:32,915 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/.tmp/hbase.id 2024-11-09T00:21:32,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741826_1002 (size=42) 2024-11-09T00:21:33,329 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/.tmp/hbase.id]:[hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/hbase.id] 2024-11-09T00:21:33,389 INFO [master/ef986b38239e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T00:21:33,396 INFO [master/ef986b38239e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T00:21:33,421 INFO [master/ef986b38239e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-09T00:21:33,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:33,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:33,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741827_1003 (size=196) 2024-11-09T00:21:33,902 INFO [master/ef986b38239e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:21:33,906 INFO [master/ef986b38239e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T00:21:33,926 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:33,933 INFO [master/ef986b38239e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T00:21:33,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741828_1004 (size=1189) 2024-11-09T00:21:34,009 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store 2024-11-09T00:21:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741829_1005 (size=34) 2024-11-09T00:21:34,438 INFO [master/ef986b38239e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T00:21:34,445 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:34,446 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T00:21:34,446 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:21:34,447 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:21:34,448 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T00:21:34,448 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:21:34,448 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:21:34,450 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731111694446Disabling compacts and flushes for region at 1731111694446Disabling writes for close at 1731111694448 (+2 ms)Writing region close event to WAL at 1731111694448Closed at 1731111694448 2024-11-09T00:21:34,452 WARN [master/ef986b38239e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/.initializing 2024-11-09T00:21:34,452 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/WALs/ef986b38239e,38689,1731111691424 2024-11-09T00:21:34,463 INFO [master/ef986b38239e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T00:21:34,482 INFO [master/ef986b38239e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef986b38239e%2C38689%2C1731111691424, suffix=, logDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/WALs/ef986b38239e,38689,1731111691424, archiveDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/oldWALs, maxLogs=10 2024-11-09T00:21:34,510 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/WALs/ef986b38239e,38689,1731111691424/ef986b38239e%2C38689%2C1731111691424.1731111694487, exclude list is [], retry=0 2024-11-09T00:21:34,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39715,DS-832d4b32-28cf-446d-a540-1a19865fc9da,DISK] 2024-11-09T00:21:34,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T00:21:34,571 INFO [master/ef986b38239e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/WALs/ef986b38239e,38689,1731111691424/ef986b38239e%2C38689%2C1731111691424.1731111694487 2024-11-09T00:21:34,572 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36429:36429)] 2024-11-09T00:21:34,572 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:21:34,572 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:34,576 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,577 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T00:21:34,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:34,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:34,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T00:21:34,668 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:34,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:34,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T00:21:34,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:34,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:34,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T00:21:34,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:34,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:34,680 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,684 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,686 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,692 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,693 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,697 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T00:21:34,703 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T00:21:34,709 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:21:34,711 INFO [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71800000, jitterRate=0.06990337371826172}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T00:21:34,719 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731111694590Initializing all the Stores at 1731111694593 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111694593Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111694594 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111694594Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111694594Cleaning up temporary data from old regions at 1731111694693 (+99 ms)Region opened successfully at 1731111694719 (+26 ms) 2024-11-09T00:21:34,721 INFO [master/ef986b38239e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T00:21:34,762 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27189791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef986b38239e/172.17.0.3:0 2024-11-09T00:21:34,800 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T00:21:34,814 INFO [master/ef986b38239e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T00:21:34,815 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T00:21:34,818 INFO [master/ef986b38239e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T00:21:34,820 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T00:21:34,826 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T00:21:34,826 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T00:21:34,863 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T00:21:34,873 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T00:21:34,923 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T00:21:34,925 INFO [master/ef986b38239e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T00:21:34,927 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T00:21:34,933 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T00:21:34,936 INFO [master/ef986b38239e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T00:21:34,941 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T00:21:34,954 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T00:21:34,957 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T00:21:34,964 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T00:21:34,984 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T00:21:34,996 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T00:21:35,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T00:21:35,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T00:21:35,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,072 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef986b38239e,38689,1731111691424, sessionid=0x1011cf81b9d0000, setting cluster-up flag (Was=false) 2024-11-09T00:21:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,155 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T00:21:35,157 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef986b38239e,38689,1731111691424 2024-11-09T00:21:35,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:35,207 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T00:21:35,208 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef986b38239e,38689,1731111691424 2024-11-09T00:21:35,216 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T00:21:35,252 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(746): ClusterId : b39df24e-75e2-4c08-8f69-2ef0b1a146a3 2024-11-09T00:21:35,256 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T00:21:35,284 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T00:21:35,284 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T00:21:35,295 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T00:21:35,295 DEBUG [RS:0;ef986b38239e:35861 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64aaaa99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef986b38239e/172.17.0.3:0 2024-11-09T00:21:35,315 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T00:21:35,318 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef986b38239e:35861 2024-11-09T00:21:35,323 INFO [RS:0;ef986b38239e:35861 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T00:21:35,323 INFO [RS:0;ef986b38239e:35861 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T00:21:35,324 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T00:21:35,327 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef986b38239e,38689,1731111691424 with port=35861, startcode=1731111692607 2024-11-09T00:21:35,328 INFO [master/ef986b38239e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T00:21:35,336 INFO [master/ef986b38239e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T00:21:35,340 DEBUG [RS:0;ef986b38239e:35861 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T00:21:35,343 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef986b38239e,38689,1731111691424 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T00:21:35,354 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef986b38239e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T00:21:35,355 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef986b38239e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T00:21:35,355 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef986b38239e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T00:21:35,355 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef986b38239e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T00:21:35,355 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef986b38239e:0, corePoolSize=10, maxPoolSize=10 2024-11-09T00:21:35,356 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,356 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef986b38239e:0, corePoolSize=2, maxPoolSize=2 2024-11-09T00:21:35,356 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,399 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T00:21:35,402 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T00:21:35,422 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:35,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39891, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T00:21:35,422 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T00:21:35,429 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731111725429 2024-11-09T00:21:35,431 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38689 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:35,434 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T00:21:35,436 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T00:21:35,439 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T00:21:35,440 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T00:21:35,441 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T00:21:35,442 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T00:21:35,448 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,464 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T00:21:35,466 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T00:21:35,466 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T00:21:35,470 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T00:21:35,470 INFO [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T00:21:35,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741831_1007 (size=1321) 2024-11-09T00:21:35,481 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T00:21:35,481 WARN [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T00:21:35,482 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.large.0-1731111695472,5,FailOnTimeoutGroup] 2024-11-09T00:21:35,483 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.small.0-1731111695482,5,FailOnTimeoutGroup] 2024-11-09T00:21:35,483 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,483 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T00:21:35,485 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,485 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,583 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef986b38239e,38689,1731111691424 with port=35861, startcode=1731111692607 2024-11-09T00:21:35,585 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38689 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,588 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38689 {}] master.ServerManager(517): Registering regionserver=ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,596 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:21:35,596 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44111 2024-11-09T00:21:35,596 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T00:21:35,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T00:21:35,644 DEBUG [RS:0;ef986b38239e:35861 {}] zookeeper.ZKUtil(111): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,644 WARN [RS:0;ef986b38239e:35861 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T00:21:35,645 INFO [RS:0;ef986b38239e:35861 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T00:21:35,645 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,646 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef986b38239e,35861,1731111692607] 2024-11-09T00:21:35,673 INFO [RS:0;ef986b38239e:35861 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T00:21:35,694 INFO [RS:0;ef986b38239e:35861 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T00:21:35,700 INFO [RS:0;ef986b38239e:35861 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T00:21:35,700 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,701 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T00:21:35,708 INFO [RS:0;ef986b38239e:35861 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T00:21:35,710 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,710 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,711 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,711 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,711 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,712 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,712 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef986b38239e:0, corePoolSize=2, maxPoolSize=2 2024-11-09T00:21:35,712 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,712 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,712 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,713 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,713 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,713 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef986b38239e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T00:21:35,713 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef986b38239e:0, corePoolSize=3, maxPoolSize=3 2024-11-09T00:21:35,714 DEBUG [RS:0;ef986b38239e:35861 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0, corePoolSize=3, maxPoolSize=3 2024-11-09T00:21:35,717 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,717 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,717 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,718 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,718 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,718 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,35861,1731111692607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T00:21:35,740 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T00:21:35,742 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,35861,1731111692607-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,742 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,743 INFO [RS:0;ef986b38239e:35861 {}] regionserver.Replication(171): ef986b38239e,35861,1731111692607 started 2024-11-09T00:21:35,763 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:35,763 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1482): Serving as ef986b38239e,35861,1731111692607, RpcServer on ef986b38239e/172.17.0.3:35861, sessionid=0x1011cf81b9d0001 2024-11-09T00:21:35,764 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T00:21:35,765 DEBUG [RS:0;ef986b38239e:35861 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,765 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef986b38239e,35861,1731111692607' 2024-11-09T00:21:35,765 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T00:21:35,766 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T00:21:35,768 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T00:21:35,768 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T00:21:35,768 DEBUG [RS:0;ef986b38239e:35861 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef986b38239e,35861,1731111692607 2024-11-09T00:21:35,768 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef986b38239e,35861,1731111692607' 2024-11-09T00:21:35,768 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T00:21:35,769 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T00:21:35,770 DEBUG [RS:0;ef986b38239e:35861 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T00:21:35,770 INFO [RS:0;ef986b38239e:35861 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T00:21:35,770 INFO [RS:0;ef986b38239e:35861 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T00:21:35,878 INFO [RS:0;ef986b38239e:35861 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T00:21:35,888 INFO [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef986b38239e%2C35861%2C1731111692607, suffix=, logDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607, archiveDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/oldWALs, maxLogs=32 2024-11-09T00:21:35,893 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T00:21:35,894 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:21:35,917 DEBUG [RS:0;ef986b38239e:35861 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607/ef986b38239e%2C35861%2C1731111692607.1731111695895, exclude list is [], retry=0 2024-11-09T00:21:35,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39715,DS-832d4b32-28cf-446d-a540-1a19865fc9da,DISK] 2024-11-09T00:21:35,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741832_1008 (size=32) 2024-11-09T00:21:35,992 INFO [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607/ef986b38239e%2C35861%2C1731111692607.1731111695895 2024-11-09T00:21:35,993 DEBUG [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36429:36429)] 2024-11-09T00:21:36,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:36,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T00:21:36,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T00:21:36,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:36,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:36,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T00:21:36,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T00:21:36,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:36,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:36,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T00:21:36,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T00:21:36,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:36,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:36,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T00:21:36,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T00:21:36,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:36,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:36,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T00:21:36,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740 2024-11-09T00:21:36,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740 2024-11-09T00:21:36,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T00:21:36,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T00:21:36,415 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:21:36,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T00:21:36,426 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:21:36,427 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58735391, jitterRate=-0.12477447092533112}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:21:36,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731111696380Initializing all the Stores at 1731111696383 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111696383Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111696383Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111696384 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111696384Cleaning up temporary data from old regions at 1731111696413 (+29 ms)Region opened successfully at 1731111696430 (+17 ms) 2024-11-09T00:21:36,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T00:21:36,431 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T00:21:36,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T00:21:36,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T00:21:36,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T00:21:36,435 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T00:21:36,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731111696430Disabling compacts and flushes for region at 1731111696430Disabling writes for close at 1731111696431 (+1 ms)Writing region close event to WAL at 1731111696434 (+3 ms)Closed at 1731111696435 (+1 ms) 2024-11-09T00:21:36,443 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T00:21:36,443 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T00:21:36,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T00:21:36,468 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T00:21:36,481 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T00:21:36,635 DEBUG [ef986b38239e:38689 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T00:21:36,650 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:21:36,658 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef986b38239e,35861,1731111692607, state=OPENING 2024-11-09T00:21:36,700 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T00:21:36,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:36,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:21:36,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T00:21:36,754 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T00:21:36,756 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T00:21:36,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:21:36,937 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T00:21:36,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42923, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T00:21:36,953 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T00:21:36,954 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T00:21:36,954 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T00:21:36,958 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef986b38239e%2C35861%2C1731111692607.meta, suffix=.meta, logDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607, archiveDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/oldWALs, maxLogs=32 2024-11-09T00:21:36,979 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607/ef986b38239e%2C35861%2C1731111692607.meta.1731111696961.meta, exclude list is [], retry=0 2024-11-09T00:21:36,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39715,DS-832d4b32-28cf-446d-a540-1a19865fc9da,DISK] 2024-11-09T00:21:36,988 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/WALs/ef986b38239e,35861,1731111692607/ef986b38239e%2C35861%2C1731111692607.meta.1731111696961.meta 2024-11-09T00:21:36,988 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36429:36429)] 2024-11-09T00:21:36,988 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:21:36,991 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T00:21:36,994 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T00:21:36,999 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T00:21:37,004 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T00:21:37,005 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:37,005 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T00:21:37,005 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T00:21:37,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T00:21:37,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T00:21:37,016 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:37,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:37,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T00:21:37,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T00:21:37,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:37,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:37,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T00:21:37,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T00:21:37,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:37,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:37,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T00:21:37,029 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T00:21:37,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:37,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T00:21:37,031 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T00:21:37,033 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740 2024-11-09T00:21:37,039 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740 2024-11-09T00:21:37,047 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T00:21:37,048 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T00:21:37,049 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:21:37,052 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T00:21:37,055 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63979867, jitterRate=-0.046625688672065735}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:21:37,055 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T00:21:37,057 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731111697006Writing region info on filesystem at 1731111697006Initializing all the Stores at 1731111697010 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111697011 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111697012 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111697012Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731111697012Cleaning up temporary data from old regions at 1731111697048 (+36 ms)Running coprocessor post-open hooks at 1731111697055 (+7 ms)Region opened successfully at 1731111697056 (+1 ms) 2024-11-09T00:21:37,065 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731111696926 2024-11-09T00:21:37,081 DEBUG [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T00:21:37,082 INFO [RS_OPEN_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T00:21:37,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:21:37,090 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef986b38239e,35861,1731111692607, state=OPEN 2024-11-09T00:21:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T00:21:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T00:21:37,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T00:21:37,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T00:21:37,189 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:37,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T00:21:37,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef986b38239e,35861,1731111692607 in 430 msec 2024-11-09T00:21:37,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T00:21:37,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 752 msec 2024-11-09T00:21:37,216 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T00:21:37,216 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T00:21:37,245 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:37,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:37,282 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:37,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57597, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:37,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.0600 sec 2024-11-09T00:21:37,319 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731111697319, completionTime=-1 2024-11-09T00:21:37,323 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T00:21:37,323 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T00:21:37,367 INFO [master/ef986b38239e:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T00:21:37,368 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731111757368 2024-11-09T00:21:37,368 INFO [master/ef986b38239e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731111817368 2024-11-09T00:21:37,368 INFO [master/ef986b38239e:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 44 msec 2024-11-09T00:21:37,372 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,373 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,373 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,375 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef986b38239e:38689, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,376 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,379 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,385 DEBUG [master/ef986b38239e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T00:21:37,416 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.585sec 2024-11-09T00:21:37,417 INFO [master/ef986b38239e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T00:21:37,419 INFO [master/ef986b38239e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T00:21:37,420 INFO [master/ef986b38239e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T00:21:37,421 INFO [master/ef986b38239e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T00:21:37,421 INFO [master/ef986b38239e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T00:21:37,422 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T00:21:37,423 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T00:21:37,444 DEBUG [master/ef986b38239e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T00:21:37,446 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T00:21:37,446 INFO [master/ef986b38239e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef986b38239e,38689,1731111691424-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T00:21:37,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:37,469 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T00:21:37,469 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T00:21:37,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:37,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:37,492 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f7ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:37,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:37,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:37,512 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:37,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:37,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:37,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef986b38239e,38689,1731111691424 2024-11-09T00:21:37,585 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=645, ProcessCount=11, AvailableMemoryMB=4084 2024-11-09T00:21:37,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:21:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:21:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7793fe81 2024-11-09T00:21:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:21:37,626 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:21:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:21:37,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:21:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:21:37,659 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:21:37,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T00:21:37,663 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:37,667 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:21:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T00:21:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741835_1011 (size=959) 2024-11-09T00:21:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T00:21:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T00:21:38,105 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:21:38,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741836_1012 (size=53) 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9e3137292ff5c555eac85d85b3bc65c6, disabling compactions & flushes 2024-11-09T00:21:38,127 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. after waiting 0 ms 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,127 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e3137292ff5c555eac85d85b3bc65c6: Waiting for close lock at 1731111698127Disabling compacts and flushes for region at 1731111698127Disabling writes for close at 1731111698127Writing region close event to WAL at 1731111698127Closed at 1731111698127 2024-11-09T00:21:38,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:21:38,138 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111698131"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111698131"}]},"ts":"1731111698131"} 2024-11-09T00:21:38,144 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:21:38,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:21:38,151 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111698148"}]},"ts":"1731111698148"} 2024-11-09T00:21:38,158 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:21:38,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, ASSIGN}] 2024-11-09T00:21:38,164 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, ASSIGN 2024-11-09T00:21:38,167 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:21:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T00:21:38,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e3137292ff5c555eac85d85b3bc65c6, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:21:38,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, ASSIGN because future has completed 2024-11-09T00:21:38,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:21:38,515 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,516 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:21:38,516 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,517 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:21:38,517 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,517 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,530 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,534 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e3137292ff5c555eac85d85b3bc65c6 columnFamilyName A 2024-11-09T00:21:38,534 DEBUG [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:38,536 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(327): Store=9e3137292ff5c555eac85d85b3bc65c6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:38,536 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,539 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e3137292ff5c555eac85d85b3bc65c6 columnFamilyName B 2024-11-09T00:21:38,539 DEBUG [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:38,540 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(327): Store=9e3137292ff5c555eac85d85b3bc65c6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:38,540 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,545 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e3137292ff5c555eac85d85b3bc65c6 columnFamilyName C 2024-11-09T00:21:38,545 DEBUG [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:21:38,546 INFO [StoreOpener-9e3137292ff5c555eac85d85b3bc65c6-1 {}] regionserver.HStore(327): Store=9e3137292ff5c555eac85d85b3bc65c6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:21:38,546 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,549 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,551 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,554 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,554 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,555 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:21:38,559 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,573 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:21:38,575 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9e3137292ff5c555eac85d85b3bc65c6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63671133, jitterRate=-0.0512261837720871}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:21:38,575 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:38,577 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9e3137292ff5c555eac85d85b3bc65c6: Running coprocessor pre-open hook at 1731111698517Writing region info on filesystem at 1731111698517Initializing all the Stores at 1731111698526 (+9 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111698526Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111698529 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111698529Cleaning up temporary data from old regions at 1731111698554 (+25 ms)Running coprocessor post-open hooks at 1731111698575 (+21 ms)Region opened successfully at 1731111698577 (+2 ms) 2024-11-09T00:21:38,580 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., pid=6, masterSystemTime=1731111698497 2024-11-09T00:21:38,585 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,585 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:38,586 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e3137292ff5c555eac85d85b3bc65c6, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:21:38,589 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38689 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=ef986b38239e,35861,1731111692607, table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T00:21:38,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:21:38,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T00:21:38,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 in 261 msec 2024-11-09T00:21:38,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T00:21:38,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, ASSIGN in 442 msec 2024-11-09T00:21:38,611 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:21:38,612 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111698611"}]},"ts":"1731111698611"} 2024-11-09T00:21:38,618 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:21:38,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:21:38,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 976 msec 2024-11-09T00:21:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T00:21:38,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:38,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75100a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cfb3d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,882 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:38,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:38,894 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:21:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:21:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7bda4d25 2024-11-09T00:21:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:21:38,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:21:38,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,955 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,976 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:38,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:38,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:38,999 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602a994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:39,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:39,005 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50131031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:39,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:39,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ee6c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:39,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:39,022 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:39,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcae178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:39,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:39,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4686c825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:39,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:21:39,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:21:39,051 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:21:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:21:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:21:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:21:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:21:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:39,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:21:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:21:39,073 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,074 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,089 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,093 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,093 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,098 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,131 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,148 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:39,186 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,188 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,245 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:21:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:21:39,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:21:39,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T00:21:39,250 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:39,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:39,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:21:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:21:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:39,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:39,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:21:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:39,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:39,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:39,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/f164a3c775b8451381315d1a934ed0f6 is 50, key is test_row_0/A:col10/1731111699310/Put/seqid=0 2024-11-09T00:21:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741837_1013 (size=14341) 2024-11-09T00:21:39,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/f164a3c775b8451381315d1a934ed0f6 2024-11-09T00:21:39,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111759558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111759566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111759574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111759581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111759589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:39,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111759715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111759727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111759727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111759728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111759730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a3101d4a02934558a4438dd1c440271e is 50, key is test_row_0/B:col10/1731111699310/Put/seqid=0 2024-11-09T00:21:39,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741838_1014 (size=12001) 2024-11-09T00:21:39,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a3101d4a02934558a4438dd1c440271e 2024-11-09T00:21:39,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d5267864280e47fe8673b3f6ecd30134 is 50, key is test_row_0/C:col10/1731111699310/Put/seqid=0 2024-11-09T00:21:39,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741839_1015 (size=12001) 2024-11-09T00:21:39,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d5267864280e47fe8673b3f6ecd30134 2024-11-09T00:21:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:39,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/f164a3c775b8451381315d1a934ed0f6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6 2024-11-09T00:21:39,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6, entries=200, sequenceid=13, filesize=14.0 K 2024-11-09T00:21:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a3101d4a02934558a4438dd1c440271e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e 2024-11-09T00:21:39,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:21:39,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d5267864280e47fe8673b3f6ecd30134 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134 2024-11-09T00:21:39,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:39,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:39,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:39,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111759938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111759947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:39,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:21:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111759949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:39,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:39,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:39,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9e3137292ff5c555eac85d85b3bc65c6 in 630ms, sequenceid=13, compaction requested=false 2024-11-09T00:21:39,982 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T00:21:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:40,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:40,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T00:21:40,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/99787711ffa14549b592c9a8e3b29295 is 50, key is test_row_0/A:col10/1731111699978/Put/seqid=0 2024-11-09T00:21:40,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741840_1016 (size=14341) 2024-11-09T00:21:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111760072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111760074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111760207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111760207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111760277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111760278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111760279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:40,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111760417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111760427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/99787711ffa14549b592c9a8e3b29295 2024-11-09T00:21:40,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/57265410573d4d568e41e6eb825fcab7 is 50, key is test_row_0/B:col10/1731111699978/Put/seqid=0 2024-11-09T00:21:40,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741841_1017 (size=12001) 2024-11-09T00:21:40,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111760736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111760736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111760796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111760808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111760805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:40,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:40,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:40,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/57265410573d4d568e41e6eb825fcab7 2024-11-09T00:21:40,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/599e57b7ff1c4e7fa030ccf477ff5410 is 50, key is test_row_0/C:col10/1731111699978/Put/seqid=0 2024-11-09T00:21:41,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741842_1018 (size=12001) 2024-11-09T00:21:41,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/599e57b7ff1c4e7fa030ccf477ff5410 2024-11-09T00:21:41,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/99787711ffa14549b592c9a8e3b29295 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295 2024-11-09T00:21:41,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295, entries=200, sequenceid=40, filesize=14.0 K 2024-11-09T00:21:41,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:41,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:41,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:41,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/57265410573d4d568e41e6eb825fcab7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7 2024-11-09T00:21:41,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T00:21:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/599e57b7ff1c4e7fa030ccf477ff5410 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410 2024-11-09T00:21:41,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T00:21:41,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9e3137292ff5c555eac85d85b3bc65c6 in 1110ms, sequenceid=40, compaction requested=false 2024-11-09T00:21:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:41,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T00:21:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:41,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T00:21:41,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d7abe171f6034a478bfdc2e3d78ff09d is 50, key is test_row_0/A:col10/1731111700067/Put/seqid=0 2024-11-09T00:21:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:41,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741843_1019 (size=12001) 2024-11-09T00:21:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111761425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111761429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111761546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111761545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,665 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T00:21:41,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d7abe171f6034a478bfdc2e3d78ff09d 2024-11-09T00:21:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e637a5a4e07a41348ef5f1ee5642e059 is 50, key is test_row_0/B:col10/1731111700067/Put/seqid=0 2024-11-09T00:21:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741844_1020 (size=12001) 2024-11-09T00:21:41,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e637a5a4e07a41348ef5f1ee5642e059 2024-11-09T00:21:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111761765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111761765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ec1113463397411997ed512ce03d63a7 is 50, key is test_row_0/C:col10/1731111700067/Put/seqid=0 2024-11-09T00:21:41,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741845_1021 (size=12001) 2024-11-09T00:21:41,800 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ec1113463397411997ed512ce03d63a7 2024-11-09T00:21:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d7abe171f6034a478bfdc2e3d78ff09d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d 2024-11-09T00:21:41,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111761815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111761825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111761826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:41,829 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,831 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,832 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:41,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T00:21:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e637a5a4e07a41348ef5f1ee5642e059 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059 2024-11-09T00:21:41,868 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T00:21:41,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ec1113463397411997ed512ce03d63a7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7 2024-11-09T00:21:41,890 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T00:21:41,893 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 9e3137292ff5c555eac85d85b3bc65c6 in 659ms, sequenceid=49, compaction requested=true 2024-11-09T00:21:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T00:21:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T00:21:41,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T00:21:41,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-09T00:21:41,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7400 sec 2024-11-09T00:21:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:42,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:42,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T00:21:42,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/ecf41c1769b942da8e17fab4ef23d010 is 50, key is test_row_0/A:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111762116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111762126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741846_1022 (size=16681) 2024-11-09T00:21:42,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/ecf41c1769b942da8e17fab4ef23d010 2024-11-09T00:21:42,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d8491b1296d143a38c7be54b1811d5fb is 50, key is test_row_0/B:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741847_1023 (size=12001) 2024-11-09T00:21:42,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d8491b1296d143a38c7be54b1811d5fb 2024-11-09T00:21:42,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111762235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111762245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/9d96e733fc6a45329ee9b91290a8e47d is 50, key is test_row_0/C:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741848_1024 (size=12001) 2024-11-09T00:21:42,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/9d96e733fc6a45329ee9b91290a8e47d 2024-11-09T00:21:42,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/ecf41c1769b942da8e17fab4ef23d010 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010 2024-11-09T00:21:42,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010, entries=250, sequenceid=78, filesize=16.3 K 2024-11-09T00:21:42,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d8491b1296d143a38c7be54b1811d5fb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb 2024-11-09T00:21:42,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T00:21:42,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/9d96e733fc6a45329ee9b91290a8e47d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d 2024-11-09T00:21:42,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T00:21:42,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 9e3137292ff5c555eac85d85b3bc65c6 in 301ms, sequenceid=78, compaction requested=true 2024-11-09T00:21:42,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:42,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:42,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:42,398 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:42,398 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:42,406 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:42,408 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:42,409 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:42,409 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=46.9 K 2024-11-09T00:21:42,410 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:42,410 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:42,410 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:42,411 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=56.0 K 2024-11-09T00:21:42,412 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a3101d4a02934558a4438dd1c440271e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111699310 2024-11-09T00:21:42,414 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting f164a3c775b8451381315d1a934ed0f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111699285 2024-11-09T00:21:42,416 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 57265410573d4d568e41e6eb825fcab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111699551 2024-11-09T00:21:42,427 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e637a5a4e07a41348ef5f1ee5642e059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731111700046 2024-11-09T00:21:42,430 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d8491b1296d143a38c7be54b1811d5fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701422 2024-11-09T00:21:42,436 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99787711ffa14549b592c9a8e3b29295, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111699551 2024-11-09T00:21:42,439 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7abe171f6034a478bfdc2e3d78ff09d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731111700046 2024-11-09T00:21:42,440 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf41c1769b942da8e17fab4ef23d010, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701394 2024-11-09T00:21:42,487 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:42,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:21:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:42,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/09b7ad5bfce34495849232614586beb7 is 50, key is test_row_0/A:col10/1731111702108/Put/seqid=0 2024-11-09T00:21:42,521 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:21:42,529 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:42,530 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a7434e6646a34beaa7e89089b6624f88 is 50, key is test_row_0/B:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,542 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#14 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:42,543 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/59288a558e304478b94e4b349e4119af is 50, key is test_row_0/A:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741850_1026 (size=12139) 2024-11-09T00:21:42,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741849_1025 (size=19021) 2024-11-09T00:21:42,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/09b7ad5bfce34495849232614586beb7 2024-11-09T00:21:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741851_1027 (size=12139) 2024-11-09T00:21:42,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/680ffff3406f46f887599cbfc71c274e is 50, key is test_row_0/B:col10/1731111702108/Put/seqid=0 2024-11-09T00:21:42,681 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/59288a558e304478b94e4b349e4119af as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/59288a558e304478b94e4b349e4119af 2024-11-09T00:21:42,684 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T00:21:42,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T00:21:42,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T00:21:42,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T00:21:42,696 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T00:21:42,696 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T00:21:42,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T00:21:42,698 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T00:21:42,712 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into 59288a558e304478b94e4b349e4119af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:42,713 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:42,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741852_1028 (size=12001) 2024-11-09T00:21:42,716 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=12, startTime=1731111702394; duration=0sec 2024-11-09T00:21:42,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:42,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:42,718 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:42,724 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:42,724 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:42,724 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:42,725 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=46.9 K 2024-11-09T00:21:42,730 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5267864280e47fe8673b3f6ecd30134, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111699310 2024-11-09T00:21:42,731 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 599e57b7ff1c4e7fa030ccf477ff5410, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111699551 2024-11-09T00:21:42,732 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1113463397411997ed512ce03d63a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731111700046 2024-11-09T00:21:42,735 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d96e733fc6a45329ee9b91290a8e47d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701422 2024-11-09T00:21:42,789 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#16 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:42,791 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/0b7e5b1b8cf0476aa86906300d39c167 is 50, key is test_row_0/C:col10/1731111702091/Put/seqid=0 2024-11-09T00:21:42,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111762781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111762793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741853_1029 (size=12139) 2024-11-09T00:21:42,862 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/0b7e5b1b8cf0476aa86906300d39c167 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0b7e5b1b8cf0476aa86906300d39c167 2024-11-09T00:21:42,892 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into 0b7e5b1b8cf0476aa86906300d39c167(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:42,892 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:42,892 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=12, startTime=1731111702398; duration=0sec 2024-11-09T00:21:42,893 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:42,893 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111762905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111762906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,052 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a7434e6646a34beaa7e89089b6624f88 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a7434e6646a34beaa7e89089b6624f88 2024-11-09T00:21:43,085 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into a7434e6646a34beaa7e89089b6624f88(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:43,085 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:43,088 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=12, startTime=1731111702398; duration=0sec 2024-11-09T00:21:43,089 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:43,090 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:43,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/680ffff3406f46f887599cbfc71c274e 2024-11-09T00:21:43,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111763115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111763115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1c494f4905a64a739408db8a0b2395eb is 50, key is test_row_0/C:col10/1731111702108/Put/seqid=0 2024-11-09T00:21:43,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741854_1030 (size=12001) 2024-11-09T00:21:43,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1c494f4905a64a739408db8a0b2395eb 2024-11-09T00:21:43,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/09b7ad5bfce34495849232614586beb7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7 2024-11-09T00:21:43,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7, entries=300, sequenceid=89, filesize=18.6 K 2024-11-09T00:21:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/680ffff3406f46f887599cbfc71c274e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e 2024-11-09T00:21:43,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T00:21:43,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1c494f4905a64a739408db8a0b2395eb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb 2024-11-09T00:21:43,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T00:21:43,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9e3137292ff5c555eac85d85b3bc65c6 in 903ms, sequenceid=89, compaction requested=false 2024-11-09T00:21:43,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T00:21:43,415 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:43,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T00:21:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:43,422 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:43,423 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:43,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:43,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:43,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T00:21:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:43,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6b92f5857aec404da6a635f2e8024895 is 50, key is test_row_0/A:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:43,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111763496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111763501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741855_1031 (size=12001) 2024-11-09T00:21:43,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6b92f5857aec404da6a635f2e8024895 2024-11-09T00:21:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:43,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/227d3420dc0f4fc88a365711ca096a79 is 50, key is test_row_0/B:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:43,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741856_1032 (size=12001) 2024-11-09T00:21:43,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/227d3420dc0f4fc88a365711ca096a79 2024-11-09T00:21:43,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ed0c00766a584a0ea8b2a7ea2b414a10 is 50, key is test_row_0/C:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:43,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111763605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111763616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:43,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741857_1033 (size=12001) 2024-11-09T00:21:43,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ed0c00766a584a0ea8b2a7ea2b414a10 2024-11-09T00:21:43,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6b92f5857aec404da6a635f2e8024895 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895 2024-11-09T00:21:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:43,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895, entries=150, sequenceid=119, filesize=11.7 K 2024-11-09T00:21:43,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:43,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/227d3420dc0f4fc88a365711ca096a79 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79 2024-11-09T00:21:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79, entries=150, sequenceid=119, filesize=11.7 K 2024-11-09T00:21:43,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ed0c00766a584a0ea8b2a7ea2b414a10 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10 2024-11-09T00:21:43,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10, entries=150, sequenceid=119, filesize=11.7 K 2024-11-09T00:21:43,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9e3137292ff5c555eac85d85b3bc65c6 in 329ms, sequenceid=119, compaction requested=true 2024-11-09T00:21:43,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:43,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:43,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:43,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:43,787 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:43,789 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:43,789 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:43,789 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,789 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/59288a558e304478b94e4b349e4119af, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=42.1 K 2024-11-09T00:21:43,793 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:43,793 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:43,793 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,794 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a7434e6646a34beaa7e89089b6624f88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=35.3 K 2024-11-09T00:21:43,794 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59288a558e304478b94e4b349e4119af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701422 2024-11-09T00:21:43,795 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a7434e6646a34beaa7e89089b6624f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701422 2024-11-09T00:21:43,797 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b7ad5bfce34495849232614586beb7, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111702108 2024-11-09T00:21:43,797 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 680ffff3406f46f887599cbfc71c274e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111702108 2024-11-09T00:21:43,798 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 227d3420dc0f4fc88a365711ca096a79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:43,798 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b92f5857aec404da6a635f2e8024895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:43,830 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#21 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:43,831 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/058863c750c3413c9128d1911eb7ab17 is 50, key is test_row_0/A:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:43,841 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#22 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:43,842 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/24221a5610204917a07bf476e032ff48 is 50, key is test_row_0/B:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:43,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:21:43,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741859_1035 (size=12241) 2024-11-09T00:21:43,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741858_1034 (size=12241) 2024-11-09T00:21:43,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/addf6c05217d42e8a4fa1684c49d4b58 is 50, key is test_row_0/A:col10/1731111703490/Put/seqid=0 2024-11-09T00:21:43,904 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/24221a5610204917a07bf476e032ff48 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/24221a5610204917a07bf476e032ff48 2024-11-09T00:21:43,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:43,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:43,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:43,927 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 24221a5610204917a07bf476e032ff48(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:43,927 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:43,928 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111703787; duration=0sec 2024-11-09T00:21:43,929 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:43,932 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741860_1036 (size=14441) 2024-11-09T00:21:43,936 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:43,938 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:43,939 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:43,939 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:43,939 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0b7e5b1b8cf0476aa86906300d39c167, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=35.3 K 2024-11-09T00:21:43,940 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7e5b1b8cf0476aa86906300d39c167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731111701422 2024-11-09T00:21:43,941 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c494f4905a64a739408db8a0b2395eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111702108 2024-11-09T00:21:43,941 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0c00766a584a0ea8b2a7ea2b414a10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:43,983 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#24 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:43,983 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/b1c80105e1a74127a5404fc8528fdfff is 50, key is test_row_0/C:col10/1731111702777/Put/seqid=0 2024-11-09T00:21:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741861_1037 (size=12241) 2024-11-09T00:21:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:44,049 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/b1c80105e1a74127a5404fc8528fdfff as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/b1c80105e1a74127a5404fc8528fdfff 2024-11-09T00:21:44,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,090 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into b1c80105e1a74127a5404fc8528fdfff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:44,090 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:44,090 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111703789; duration=0sec 2024-11-09T00:21:44,091 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:44,091 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111764097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111764107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111764113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111764122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111764128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111764235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111764236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111764246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111764246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111764255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,304 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/058863c750c3413c9128d1911eb7ab17 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/058863c750c3413c9128d1911eb7ab17 2024-11-09T00:21:44,326 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into 058863c750c3413c9128d1911eb7ab17(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:44,326 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:44,327 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111703786; duration=0sec 2024-11-09T00:21:44,327 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:44,327 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:44,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/addf6c05217d42e8a4fa1684c49d4b58 2024-11-09T00:21:44,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a531f96b2ca24b149a602f8396f3cef7 is 50, key is test_row_0/B:col10/1731111703490/Put/seqid=0 2024-11-09T00:21:44,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741862_1038 (size=12101) 2024-11-09T00:21:44,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a531f96b2ca24b149a602f8396f3cef7 2024-11-09T00:21:44,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/287494d0abdb40758e8e840a2a123c7d is 50, key is test_row_0/C:col10/1731111703490/Put/seqid=0 2024-11-09T00:21:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111764456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111764456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111764456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111764466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111764475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741863_1039 (size=12101) 2024-11-09T00:21:44,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:44,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111764765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111764775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111764776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111764785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111764785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:44,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:44,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:44,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/287494d0abdb40758e8e840a2a123c7d 2024-11-09T00:21:44,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/addf6c05217d42e8a4fa1684c49d4b58 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58 2024-11-09T00:21:44,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58, entries=200, sequenceid=130, filesize=14.1 K 2024-11-09T00:21:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a531f96b2ca24b149a602f8396f3cef7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7 2024-11-09T00:21:44,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7, entries=150, sequenceid=130, filesize=11.8 K 2024-11-09T00:21:44,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/287494d0abdb40758e8e840a2a123c7d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d 2024-11-09T00:21:44,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d, entries=150, sequenceid=130, filesize=11.8 K 2024-11-09T00:21:44,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9e3137292ff5c555eac85d85b3bc65c6 in 1136ms, sequenceid=130, compaction requested=false 2024-11-09T00:21:44,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:45,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T00:21:45,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:21:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/7395e276c01643eabbe4e6b6baa07afe is 50, key is test_row_0/A:col10/1731111704094/Put/seqid=0 2024-11-09T00:21:45,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741864_1040 (size=12151) 2024-11-09T00:21:45,076 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/7395e276c01643eabbe4e6b6baa07afe 2024-11-09T00:21:45,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/2abc9aebb61c485f9af1cf54bbcac214 is 50, key is test_row_0/B:col10/1731111704094/Put/seqid=0 2024-11-09T00:21:45,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741865_1041 (size=12151) 2024-11-09T00:21:45,132 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/2abc9aebb61c485f9af1cf54bbcac214 2024-11-09T00:21:45,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a29d0b52a75b4a738c6ed1ec17e58341 is 50, key is test_row_0/C:col10/1731111704094/Put/seqid=0 2024-11-09T00:21:45,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741866_1042 (size=12151) 2024-11-09T00:21:45,183 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a29d0b52a75b4a738c6ed1ec17e58341 2024-11-09T00:21:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/7395e276c01643eabbe4e6b6baa07afe as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe 2024-11-09T00:21:45,224 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe, entries=150, sequenceid=158, filesize=11.9 K 2024-11-09T00:21:45,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/2abc9aebb61c485f9af1cf54bbcac214 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214 2024-11-09T00:21:45,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214, entries=150, sequenceid=158, filesize=11.9 K 2024-11-09T00:21:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a29d0b52a75b4a738c6ed1ec17e58341 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341 2024-11-09T00:21:45,265 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341, entries=150, sequenceid=158, filesize=11.9 K 2024-11-09T00:21:45,269 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 9e3137292ff5c555eac85d85b3bc65c6 in 237ms, sequenceid=158, compaction requested=true 2024-11-09T00:21:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T00:21:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T00:21:45,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T00:21:45,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-09T00:21:45,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8670 sec 2024-11-09T00:21:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:45,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:21:45,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e4e7b16e4e0d42c7beb2d4f6869a628f is 50, key is test_row_0/A:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741867_1043 (size=12151) 2024-11-09T00:21:45,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e4e7b16e4e0d42c7beb2d4f6869a628f 2024-11-09T00:21:45,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/0f20a1636d654b5ab9ce113a2a563fa3 is 50, key is test_row_0/B:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741868_1044 (size=12151) 2024-11-09T00:21:45,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/0f20a1636d654b5ab9ce113a2a563fa3 2024-11-09T00:21:45,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111765409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111765410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111765413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111765416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111765414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/249326964fad4c118191d07b3b19ef49 is 50, key is test_row_0/C:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741869_1045 (size=12151) 2024-11-09T00:21:45,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/249326964fad4c118191d07b3b19ef49 2024-11-09T00:21:45,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e4e7b16e4e0d42c7beb2d4f6869a628f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f 2024-11-09T00:21:45,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T00:21:45,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/0f20a1636d654b5ab9ce113a2a563fa3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3 2024-11-09T00:21:45,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T00:21:45,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/249326964fad4c118191d07b3b19ef49 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49 2024-11-09T00:21:45,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49, entries=150, sequenceid=169, filesize=11.9 K 2024-11-09T00:21:45,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9e3137292ff5c555eac85d85b3bc65c6 in 215ms, sequenceid=169, compaction requested=true 2024-11-09T00:21:45,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:45,537 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:45,540 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50984 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:45,540 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:45,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:45,541 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:45,541 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,541 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/058863c750c3413c9128d1911eb7ab17, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=49.8 K 2024-11-09T00:21:45,542 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 058863c750c3413c9128d1911eb7ab17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:45,547 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting addf6c05217d42e8a4fa1684c49d4b58, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731111703490 2024-11-09T00:21:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:45,547 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:45,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T00:21:45,548 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7395e276c01643eabbe4e6b6baa07afe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731111704007 2024-11-09T00:21:45,549 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:45,549 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:45,549 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,549 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/24221a5610204917a07bf476e032ff48, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=47.5 K 2024-11-09T00:21:45,550 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e7b16e4e0d42c7beb2d4f6869a628f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:45,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24221a5610204917a07bf476e032ff48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:45,553 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a531f96b2ca24b149a602f8396f3cef7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731111703490 2024-11-09T00:21:45,553 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abc9aebb61c485f9af1cf54bbcac214, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731111704007 2024-11-09T00:21:45,555 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f20a1636d654b5ab9ce113a2a563fa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T00:21:45,565 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:45,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T00:21:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:45,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:45,572 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:45,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111765551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111765554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/dfa4b04582ee44a7bbc11ff463f9f13d is 50, key is test_row_0/A:col10/1731111705544/Put/seqid=0 2024-11-09T00:21:45,593 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#34 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:45,593 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/2782badfa3124cdca7e5307fae687b71 is 50, key is test_row_0/B:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,595 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:45,598 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/35e3ab41d8984b78b79c10b3b0dd4550 is 50, key is test_row_0/A:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111765577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111765588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111765597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741871_1047 (size=12527) 2024-11-09T00:21:45,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741872_1048 (size=12527) 2024-11-09T00:21:45,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741870_1046 (size=16931) 2024-11-09T00:21:45,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/dfa4b04582ee44a7bbc11ff463f9f13d 2024-11-09T00:21:45,659 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/35e3ab41d8984b78b79c10b3b0dd4550 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/35e3ab41d8984b78b79c10b3b0dd4550 2024-11-09T00:21:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:45,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/6b7d439afe444dc5963a1d93f220c315 is 50, key is test_row_0/B:col10/1731111705544/Put/seqid=0 2024-11-09T00:21:45,681 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into 35e3ab41d8984b78b79c10b3b0dd4550(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:45,681 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:45,681 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=12, startTime=1731111705537; duration=0sec 2024-11-09T00:21:45,681 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:45,681 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:45,681 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:45,684 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:45,684 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:45,684 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,684 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/b1c80105e1a74127a5404fc8528fdfff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=47.5 K 2024-11-09T00:21:45,686 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c80105e1a74127a5404fc8528fdfff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731111702777 2024-11-09T00:21:45,687 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 287494d0abdb40758e8e840a2a123c7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731111703490 2024-11-09T00:21:45,688 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a29d0b52a75b4a738c6ed1ec17e58341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731111704007 2024-11-09T00:21:45,689 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 249326964fad4c118191d07b3b19ef49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111765695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111765695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111765704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111765705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111765705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,719 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:45,720 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/446321c559ec41f5b25f524bb75c4bb8 is 50, key is test_row_0/C:col10/1731111705308/Put/seqid=0 2024-11-09T00:21:45,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741874_1050 (size=12527) 2024-11-09T00:21:45,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:45,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:45,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,759 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/446321c559ec41f5b25f524bb75c4bb8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/446321c559ec41f5b25f524bb75c4bb8 2024-11-09T00:21:45,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741873_1049 (size=12151) 2024-11-09T00:21:45,780 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into 446321c559ec41f5b25f524bb75c4bb8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:45,780 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:45,780 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=12, startTime=1731111705540; duration=0sec 2024-11-09T00:21:45,781 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:45,781 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:45,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:45,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111765905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111765905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111765925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111765927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111765927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,039 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/2782badfa3124cdca7e5307fae687b71 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2782badfa3124cdca7e5307fae687b71 2024-11-09T00:21:46,056 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 2782badfa3124cdca7e5307fae687b71(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:46,056 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:46,056 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=12, startTime=1731111705540; duration=0sec 2024-11-09T00:21:46,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,058 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:46,058 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:46,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/6b7d439afe444dc5963a1d93f220c315 2024-11-09T00:21:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:46,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a74850aae0d14526a0d269bd423214b6 is 50, key is test_row_0/C:col10/1731111705544/Put/seqid=0 2024-11-09T00:21:46,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111766215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111766225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111766235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111766235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111766235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741875_1051 (size=12151) 2024-11-09T00:21:46,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a74850aae0d14526a0d269bd423214b6 2024-11-09T00:21:46,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/dfa4b04582ee44a7bbc11ff463f9f13d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d 2024-11-09T00:21:46,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d, entries=250, sequenceid=197, filesize=16.5 K 2024-11-09T00:21:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/6b7d439afe444dc5963a1d93f220c315 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315 2024-11-09T00:21:46,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T00:21:46,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a74850aae0d14526a0d269bd423214b6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6 2024-11-09T00:21:46,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T00:21:46,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9e3137292ff5c555eac85d85b3bc65c6 in 1156ms, sequenceid=197, compaction requested=false 2024-11-09T00:21:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:46,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:46,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:21:46,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/2ff6573eac9449cc984ffc412d91bc30 is 50, key is test_row_0/A:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:46,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741876_1052 (size=14541) 2024-11-09T00:21:46,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/2ff6573eac9449cc984ffc412d91bc30 2024-11-09T00:21:46,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/745e4ff8217e488eaad224b0fe5a918f is 50, key is test_row_0/B:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:46,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111766812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111766823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111766824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111766825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111766827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:46,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741877_1053 (size=12151) 2024-11-09T00:21:46,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/745e4ff8217e488eaad224b0fe5a918f 2024-11-09T00:21:46,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/bb1abfaacfa541ec803cb5eea65a965c is 50, key is test_row_0/C:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741878_1054 (size=12151) 2024-11-09T00:21:46,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/bb1abfaacfa541ec803cb5eea65a965c 2024-11-09T00:21:46,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/2ff6573eac9449cc984ffc412d91bc30 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30 2024-11-09T00:21:46,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30, entries=200, sequenceid=213, filesize=14.2 K 2024-11-09T00:21:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/745e4ff8217e488eaad224b0fe5a918f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f 2024-11-09T00:21:46,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111766935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111766935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111766935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T00:21:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111766946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111766946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/bb1abfaacfa541ec803cb5eea65a965c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c 2024-11-09T00:21:46,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T00:21:46,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 9e3137292ff5c555eac85d85b3bc65c6 in 223ms, sequenceid=213, compaction requested=true 2024-11-09T00:21:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:46,975 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:46,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:46,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T00:21:46,976 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:46,979 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:46,979 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:46,979 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:46,979 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:46,980 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,980 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,980 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2782badfa3124cdca7e5307fae687b71, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.0 K 2024-11-09T00:21:46,980 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/35e3ab41d8984b78b79c10b3b0dd4550, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=43.0 K 2024-11-09T00:21:46,982 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2782badfa3124cdca7e5307fae687b71, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:46,982 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e3ab41d8984b78b79c10b3b0dd4550, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:46,983 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b7d439afe444dc5963a1d93f220c315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731111705410 2024-11-09T00:21:46,983 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa4b04582ee44a7bbc11ff463f9f13d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731111705397 2024-11-09T00:21:46,983 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 745e4ff8217e488eaad224b0fe5a918f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:46,983 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff6573eac9449cc984ffc412d91bc30, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:46,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T00:21:46,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:46,993 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:21:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/3508cba50b974214afd7d57a04ce17d9 is 50, key is test_row_0/A:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:47,018 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:47,018 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/b29133ed6adf41feb3845e3f754aa631 is 50, key is test_row_0/A:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:47,019 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#43 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:47,019 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cd6480658634435a98128182fa7b148c is 50, key is test_row_0/B:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:47,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741879_1055 (size=12151) 2024-11-09T00:21:47,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741880_1056 (size=12629) 2024-11-09T00:21:47,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741881_1057 (size=12629) 2024-11-09T00:21:47,064 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cd6480658634435a98128182fa7b148c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cd6480658634435a98128182fa7b148c 2024-11-09T00:21:47,067 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/b29133ed6adf41feb3845e3f754aa631 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b29133ed6adf41feb3845e3f754aa631 2024-11-09T00:21:47,085 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into cd6480658634435a98128182fa7b148c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:47,085 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111706975; duration=0sec 2024-11-09T00:21:47,085 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into b29133ed6adf41feb3845e3f754aa631(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:47,085 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111706975; duration=0sec 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:47,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:47,088 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:47,088 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:47,088 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:47,088 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/446321c559ec41f5b25f524bb75c4bb8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.0 K 2024-11-09T00:21:47,090 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 446321c559ec41f5b25f524bb75c4bb8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731111705301 2024-11-09T00:21:47,090 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a74850aae0d14526a0d269bd423214b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731111705410 2024-11-09T00:21:47,095 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1abfaacfa541ec803cb5eea65a965c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:47,112 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:47,113 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a0f741e2263240be9462e2cd3333c74a is 50, key is test_row_0/C:col10/1731111706746/Put/seqid=0 2024-11-09T00:21:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741882_1058 (size=12629) 2024-11-09T00:21:47,143 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a0f741e2263240be9462e2cd3333c74a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a0f741e2263240be9462e2cd3333c74a 2024-11-09T00:21:47,159 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into a0f741e2263240be9462e2cd3333c74a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:47,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:47,159 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111706975; duration=0sec 2024-11-09T00:21:47,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:47,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:47,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111767206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111767206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111767210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111767211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111767210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111767315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111767316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111767324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111767325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111767325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/3508cba50b974214afd7d57a04ce17d9 2024-11-09T00:21:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/67653ab66bc943549c0f03225c47d24f is 50, key is test_row_0/B:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:47,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741883_1059 (size=12151) 2024-11-09T00:21:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111767524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111767525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111767534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111767535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111767535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111767835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111767835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111767845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111767845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111767846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:47,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/67653ab66bc943549c0f03225c47d24f 2024-11-09T00:21:47,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/acf396ebea584cff826cba0b0f28a53e is 50, key is test_row_0/C:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:47,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741884_1060 (size=12151) 2024-11-09T00:21:47,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/acf396ebea584cff826cba0b0f28a53e 2024-11-09T00:21:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/3508cba50b974214afd7d57a04ce17d9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9 2024-11-09T00:21:47,930 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T00:21:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/67653ab66bc943549c0f03225c47d24f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f 2024-11-09T00:21:47,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T00:21:47,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/acf396ebea584cff826cba0b0f28a53e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e 2024-11-09T00:21:47,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T00:21:47,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9e3137292ff5c555eac85d85b3bc65c6 in 970ms, sequenceid=238, compaction requested=false 2024-11-09T00:21:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T00:21:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T00:21:47,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T00:21:47,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3950 sec 2024-11-09T00:21:47,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4060 sec 2024-11-09T00:21:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:48,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:48,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:21:48,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/debccfc7a9aa40c4a2834465e5e8ec8b is 50, key is test_row_1/A:col10/1731111708345/Put/seqid=0 2024-11-09T00:21:48,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741885_1061 (size=9757) 2024-11-09T00:21:48,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/debccfc7a9aa40c4a2834465e5e8ec8b 2024-11-09T00:21:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111768398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111768399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a8cfc574cc8442f2bc89673908d9aec8 is 50, key is test_row_1/B:col10/1731111708345/Put/seqid=0 2024-11-09T00:21:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111768401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111768401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111768404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741886_1062 (size=9757) 2024-11-09T00:21:48,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111768515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111768515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111768516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111768516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111768526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111768725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111768725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111768726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111768726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111768735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:48,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a8cfc574cc8442f2bc89673908d9aec8 2024-11-09T00:21:48,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/7dc1e4d29d50487c8241ad7055aeefc9 is 50, key is test_row_1/C:col10/1731111708345/Put/seqid=0 2024-11-09T00:21:48,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741887_1063 (size=9757) 2024-11-09T00:21:48,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/7dc1e4d29d50487c8241ad7055aeefc9 2024-11-09T00:21:48,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/debccfc7a9aa40c4a2834465e5e8ec8b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b 2024-11-09T00:21:48,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b, entries=100, sequenceid=253, filesize=9.5 K 2024-11-09T00:21:48,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/a8cfc574cc8442f2bc89673908d9aec8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8 2024-11-09T00:21:48,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8, entries=100, sequenceid=253, filesize=9.5 K 2024-11-09T00:21:48,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/7dc1e4d29d50487c8241ad7055aeefc9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9 2024-11-09T00:21:48,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9, entries=100, sequenceid=253, filesize=9.5 K 2024-11-09T00:21:48,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9e3137292ff5c555eac85d85b3bc65c6 in 593ms, sequenceid=253, compaction requested=true 2024-11-09T00:21:48,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:48,941 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:48,941 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:48,942 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:48,942 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:48,942 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:48,942 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:48,943 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cd6480658634435a98128182fa7b148c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=33.7 K 2024-11-09T00:21:48,943 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:48,943 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:48,943 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b29133ed6adf41feb3845e3f754aa631, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=33.7 K 2024-11-09T00:21:48,943 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6480658634435a98128182fa7b148c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:48,944 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29133ed6adf41feb3845e3f754aa631, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:48,944 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 67653ab66bc943549c0f03225c47d24f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111706817 2024-11-09T00:21:48,944 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3508cba50b974214afd7d57a04ce17d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111706817 2024-11-09T00:21:48,945 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a8cfc574cc8442f2bc89673908d9aec8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111707210 2024-11-09T00:21:48,946 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting debccfc7a9aa40c4a2834465e5e8ec8b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111707210 2024-11-09T00:21:48,967 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:48,968 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/7bcb07785e8b48909a9e1f0e53d579c3 is 50, key is test_row_0/B:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:48,969 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:48,970 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/a1582ce1cc5940d2a9ade8d98e82f309 is 50, key is test_row_0/A:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741888_1064 (size=12731) 2024-11-09T00:21:48,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741889_1065 (size=12731) 2024-11-09T00:21:49,009 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/a1582ce1cc5940d2a9ade8d98e82f309 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a1582ce1cc5940d2a9ade8d98e82f309 2024-11-09T00:21:49,021 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into a1582ce1cc5940d2a9ade8d98e82f309(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:49,021 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:49,021 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111708940; duration=0sec 2024-11-09T00:21:49,022 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:49,022 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:49,022 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:49,025 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:49,025 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:49,025 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:49,025 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a0f741e2263240be9462e2cd3333c74a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=33.7 K 2024-11-09T00:21:49,026 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f741e2263240be9462e2cd3333c74a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111706746 2024-11-09T00:21:49,026 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting acf396ebea584cff826cba0b0f28a53e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111706817 2024-11-09T00:21:49,027 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc1e4d29d50487c8241ad7055aeefc9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111707210 2024-11-09T00:21:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:49,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:49,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:21:49,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/afba5a349def4e05a5326053571a7525 is 50, key is test_row_0/A:col10/1731111708399/Put/seqid=0 2024-11-09T00:21:49,060 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:49,061 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/97218660b26f4e91b37d79ae070b5cd0 is 50, key is test_row_0/C:col10/1731111706817/Put/seqid=0 2024-11-09T00:21:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111769056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111769059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111769060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111769062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111769062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741890_1066 (size=12301) 2024-11-09T00:21:49,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741891_1067 (size=12731) 2024-11-09T00:21:49,116 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/97218660b26f4e91b37d79ae070b5cd0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/97218660b26f4e91b37d79ae070b5cd0 2024-11-09T00:21:49,128 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into 97218660b26f4e91b37d79ae070b5cd0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:49,128 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:49,128 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111708941; duration=0sec 2024-11-09T00:21:49,129 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:49,129 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111769175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111769176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111769176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111769176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111769176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111769385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111769385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111769385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111769385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111769386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/7bcb07785e8b48909a9e1f0e53d579c3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/7bcb07785e8b48909a9e1f0e53d579c3 2024-11-09T00:21:49,421 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 7bcb07785e8b48909a9e1f0e53d579c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:49,421 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:49,421 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111708941; duration=0sec 2024-11-09T00:21:49,422 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:49,422 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:49,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/afba5a349def4e05a5326053571a7525 2024-11-09T00:21:49,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cf2168c3ff964b1d8b3b95674097241a is 50, key is test_row_0/B:col10/1731111708399/Put/seqid=0 2024-11-09T00:21:49,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741892_1068 (size=12301) 2024-11-09T00:21:49,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cf2168c3ff964b1d8b3b95674097241a 2024-11-09T00:21:49,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/c4f81f01652d4c0f855bd57ceb7878c0 is 50, key is test_row_0/C:col10/1731111708399/Put/seqid=0 2024-11-09T00:21:49,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741893_1069 (size=12301) 2024-11-09T00:21:49,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/c4f81f01652d4c0f855bd57ceb7878c0 2024-11-09T00:21:49,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/afba5a349def4e05a5326053571a7525 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525 2024-11-09T00:21:49,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T00:21:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cf2168c3ff964b1d8b3b95674097241a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a 2024-11-09T00:21:49,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T00:21:49,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/c4f81f01652d4c0f855bd57ceb7878c0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0 2024-11-09T00:21:49,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0, entries=150, sequenceid=280, filesize=12.0 K 2024-11-09T00:21:49,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 9e3137292ff5c555eac85d85b3bc65c6 in 554ms, sequenceid=280, compaction requested=false 2024-11-09T00:21:49,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:49,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:49,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:21:49,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e6b7f00a9001490886c0fa05a0d45051 is 50, key is test_row_0/A:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741894_1070 (size=12301) 2024-11-09T00:21:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T00:21:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111769731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111769732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111769733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T00:21:49,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111769735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:49,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:49,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:49,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:49,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111769736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:49,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111769845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111769846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111769846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111769846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111769854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:49,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:49,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111770055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111770056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111770056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111770065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111770065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e6b7f00a9001490886c0fa05a0d45051 2024-11-09T00:21:50,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/22b364bc9fab492c8e99ba774f6efb29 is 50, key is test_row_0/B:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:50,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741895_1071 (size=12301) 2024-11-09T00:21:50,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111770366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111770366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111770375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111770375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111770376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:50,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/22b364bc9fab492c8e99ba774f6efb29 2024-11-09T00:21:50,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/0f9ef39705f648dea1cd91c0fdc4ad8f is 50, key is test_row_0/C:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:50,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741896_1072 (size=12301) 2024-11-09T00:21:50,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/0f9ef39705f648dea1cd91c0fdc4ad8f 2024-11-09T00:21:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e6b7f00a9001490886c0fa05a0d45051 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051 2024-11-09T00:21:50,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051, entries=150, sequenceid=295, filesize=12.0 K 2024-11-09T00:21:50,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/22b364bc9fab492c8e99ba774f6efb29 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29 2024-11-09T00:21:50,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29, entries=150, sequenceid=295, filesize=12.0 K 2024-11-09T00:21:50,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/0f9ef39705f648dea1cd91c0fdc4ad8f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f 2024-11-09T00:21:50,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-09T00:21:50,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9e3137292ff5c555eac85d85b3bc65c6 in 912ms, sequenceid=295, compaction requested=true 2024-11-09T00:21:50,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:50,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:50,611 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:50,612 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:50,613 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:50,613 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:50,613 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:50,613 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:50,613 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,613 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,613 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/7bcb07785e8b48909a9e1f0e53d579c3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.5 K 2024-11-09T00:21:50,613 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a1582ce1cc5940d2a9ade8d98e82f309, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.5 K 2024-11-09T00:21:50,614 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1582ce1cc5940d2a9ade8d98e82f309, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111706817 2024-11-09T00:21:50,614 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bcb07785e8b48909a9e1f0e53d579c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111706817 2024-11-09T00:21:50,615 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting afba5a349def4e05a5326053571a7525, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731111708395 2024-11-09T00:21:50,615 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2168c3ff964b1d8b3b95674097241a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731111708395 2024-11-09T00:21:50,615 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b7f00a9001490886c0fa05a0d45051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:50,615 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b364bc9fab492c8e99ba774f6efb29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:50,627 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:50,627 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:50,628 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/b0ae80b5c4ef4ea5b95f926093947e76 is 50, key is test_row_0/A:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:50,628 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/0377977571e9453fb39d0574d548c09a is 50, key is test_row_0/B:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:50,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741897_1073 (size=12983) 2024-11-09T00:21:50,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741898_1074 (size=12983) 2024-11-09T00:21:50,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T00:21:50,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:50,683 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:21:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/554ca2672cc944beac6029c2f95af735 is 50, key is test_row_0/A:col10/1731111709733/Put/seqid=0 2024-11-09T00:21:50,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741899_1075 (size=12301) 2024-11-09T00:21:50,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/554ca2672cc944beac6029c2f95af735 2024-11-09T00:21:50,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d1cfbe972a21458da653d3de7369c6fc is 50, key is test_row_0/B:col10/1731111709733/Put/seqid=0 2024-11-09T00:21:50,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741900_1076 (size=12301) 2024-11-09T00:21:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:50,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111770892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111770892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111770897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111770899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111770900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111771004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111771005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111771005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111771015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111771015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,041 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/0377977571e9453fb39d0574d548c09a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0377977571e9453fb39d0574d548c09a 2024-11-09T00:21:51,054 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/b0ae80b5c4ef4ea5b95f926093947e76 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b0ae80b5c4ef4ea5b95f926093947e76 2024-11-09T00:21:51,054 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 0377977571e9453fb39d0574d548c09a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:51,054 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:51,054 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111710612; duration=0sec 2024-11-09T00:21:51,054 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:51,054 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:51,055 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:51,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:51,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:51,056 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:51,058 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/97218660b26f4e91b37d79ae070b5cd0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.5 K 2024-11-09T00:21:51,058 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 97218660b26f4e91b37d79ae070b5cd0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731111706817 2024-11-09T00:21:51,059 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f81f01652d4c0f855bd57ceb7878c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731111708395 2024-11-09T00:21:51,060 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9ef39705f648dea1cd91c0fdc4ad8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:51,071 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into b0ae80b5c4ef4ea5b95f926093947e76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:51,071 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:51,071 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111710611; duration=0sec 2024-11-09T00:21:51,071 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:51,071 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:51,087 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:51,088 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/61172c25bac04ce6ac25e4d001b7edf6 is 50, key is test_row_0/C:col10/1731111709056/Put/seqid=0 2024-11-09T00:21:51,131 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d1cfbe972a21458da653d3de7369c6fc 2024-11-09T00:21:51,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741901_1077 (size=12983) 2024-11-09T00:21:51,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/f2af9b4eee534d1cb19160f6e246df88 is 50, key is test_row_0/C:col10/1731111709733/Put/seqid=0 2024-11-09T00:21:51,151 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/61172c25bac04ce6ac25e4d001b7edf6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/61172c25bac04ce6ac25e4d001b7edf6 2024-11-09T00:21:51,159 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into 61172c25bac04ce6ac25e4d001b7edf6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:51,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:51,159 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111710612; duration=0sec 2024-11-09T00:21:51,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:51,159 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:51,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741902_1078 (size=12301) 2024-11-09T00:21:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/f2af9b4eee534d1cb19160f6e246df88 2024-11-09T00:21:51,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/554ca2672cc944beac6029c2f95af735 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735 2024-11-09T00:21:51,206 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735, entries=150, sequenceid=318, filesize=12.0 K 2024-11-09T00:21:51,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d1cfbe972a21458da653d3de7369c6fc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc 2024-11-09T00:21:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111771216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111771216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,220 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc, entries=150, sequenceid=318, filesize=12.0 K 2024-11-09T00:21:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111771216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/f2af9b4eee534d1cb19160f6e246df88 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88 2024-11-09T00:21:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111771225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111771225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88, entries=150, sequenceid=318, filesize=12.0 K 2024-11-09T00:21:51,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9e3137292ff5c555eac85d85b3bc65c6 in 551ms, sequenceid=318, compaction requested=false 2024-11-09T00:21:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T00:21:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T00:21:51,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T00:21:51,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4980 sec 2024-11-09T00:21:51,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5120 sec 2024-11-09T00:21:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:51,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T00:21:51,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6fe2e6d88327408c90d6ee6eb4272287 is 50, key is test_row_0/A:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:51,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741903_1079 (size=14741) 2024-11-09T00:21:51,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111771577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111771581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111771583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111771587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111771594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111771696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111771697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111771705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111771705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111771708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T00:21:51,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:51,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T00:21:51,903 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:51,905 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:51,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111771904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111771905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111771915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111771915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111771925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:51,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6fe2e6d88327408c90d6ee6eb4272287 2024-11-09T00:21:51,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/4972ee70b3c741d6b5a08e5b89d2475a is 50, key is test_row_0/B:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:52,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741904_1080 (size=12301) 2024-11-09T00:21:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:52,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/4972ee70b3c741d6b5a08e5b89d2475a 2024-11-09T00:21:52,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/22274712de65451b888039a04fc7db61 is 50, key is test_row_0/C:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:52,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741905_1081 (size=12301) 2024-11-09T00:21:52,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T00:21:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111772215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111772215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T00:21:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111772225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111772225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111772235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T00:21:52,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:52,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:52,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/22274712de65451b888039a04fc7db61 2024-11-09T00:21:52,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6fe2e6d88327408c90d6ee6eb4272287 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287 2024-11-09T00:21:52,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287, entries=200, sequenceid=336, filesize=14.4 K 2024-11-09T00:21:52,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/4972ee70b3c741d6b5a08e5b89d2475a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a 2024-11-09T00:21:52,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T00:21:52,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/22274712de65451b888039a04fc7db61 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61 2024-11-09T00:21:52,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T00:21:52,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 9e3137292ff5c555eac85d85b3bc65c6 in 986ms, sequenceid=336, compaction requested=true 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:52,512 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:52,512 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:52,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:52,513 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:52,513 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:52,513 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:52,513 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:52,513 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,513 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,514 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0377977571e9453fb39d0574d548c09a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.7 K 2024-11-09T00:21:52,514 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b0ae80b5c4ef4ea5b95f926093947e76, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=39.1 K 2024-11-09T00:21:52,514 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0377977571e9453fb39d0574d548c09a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:52,514 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ae80b5c4ef4ea5b95f926093947e76, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:52,515 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d1cfbe972a21458da653d3de7369c6fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731111709714 2024-11-09T00:21:52,515 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4972ee70b3c741d6b5a08e5b89d2475a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:52,516 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554ca2672cc944beac6029c2f95af735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731111709714 2024-11-09T00:21:52,516 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fe2e6d88327408c90d6ee6eb4272287, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:52,540 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:52,541 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/c07c908e6e584f4ab9f0dd981b175d8f is 50, key is test_row_0/B:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:52,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T00:21:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T00:21:52,550 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:52,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/a35a05a0601f4d8386e12fd60bbecbc5 is 50, key is test_row_0/A:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6e74637c59104ce29f69783c20d782cd is 50, key is test_row_0/A:col10/1731111711584/Put/seqid=0 2024-11-09T00:21:52,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741906_1082 (size=13085) 2024-11-09T00:21:52,565 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/c07c908e6e584f4ab9f0dd981b175d8f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c07c908e6e584f4ab9f0dd981b175d8f 2024-11-09T00:21:52,574 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into c07c908e6e584f4ab9f0dd981b175d8f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:52,574 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:52,574 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111712512; duration=0sec 2024-11-09T00:21:52,574 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:52,574 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:52,574 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:52,581 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:52,581 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:52,581 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:52,582 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/61172c25bac04ce6ac25e4d001b7edf6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.7 K 2024-11-09T00:21:52,582 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 61172c25bac04ce6ac25e4d001b7edf6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111709056 2024-11-09T00:21:52,583 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f2af9b4eee534d1cb19160f6e246df88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731111709714 2024-11-09T00:21:52,583 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 22274712de65451b888039a04fc7db61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:52,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741907_1083 (size=12301) 2024-11-09T00:21:52,616 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:52,616 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/eb49ea191a564b5086e42685f42fdc3d is 50, key is test_row_0/C:col10/1731111711527/Put/seqid=0 2024-11-09T00:21:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741908_1084 (size=13085) 2024-11-09T00:21:52,629 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/a35a05a0601f4d8386e12fd60bbecbc5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a35a05a0601f4d8386e12fd60bbecbc5 2024-11-09T00:21:52,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741909_1085 (size=13085) 2024-11-09T00:21:52,640 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into a35a05a0601f4d8386e12fd60bbecbc5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:52,640 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:52,640 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111712512; duration=0sec 2024-11-09T00:21:52,640 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:52,640 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:52,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111772743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111772745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111772747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111772749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111772755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111772854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111772854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111772869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111772870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,012 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6e74637c59104ce29f69783c20d782cd 2024-11-09T00:21:53,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/950947bd47884fcdafe054577e1b6648 is 50, key is test_row_0/B:col10/1731111711584/Put/seqid=0 2024-11-09T00:21:53,042 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/eb49ea191a564b5086e42685f42fdc3d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eb49ea191a564b5086e42685f42fdc3d 2024-11-09T00:21:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:53,055 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into eb49ea191a564b5086e42685f42fdc3d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:53,055 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:53,055 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111712512; duration=0sec 2024-11-09T00:21:53,055 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:53,055 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111773065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111773074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111773074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741910_1086 (size=12301) 2024-11-09T00:21:53,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111773085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111773375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111773385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111773385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111773394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,488 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/950947bd47884fcdafe054577e1b6648 2024-11-09T00:21:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d794fbc6bc834420856e1d5f3f650c0e is 50, key is test_row_0/C:col10/1731111711584/Put/seqid=0 2024-11-09T00:21:53,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741911_1087 (size=12301) 2024-11-09T00:21:53,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d794fbc6bc834420856e1d5f3f650c0e 2024-11-09T00:21:53,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/6e74637c59104ce29f69783c20d782cd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd 2024-11-09T00:21:53,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T00:21:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/950947bd47884fcdafe054577e1b6648 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648 2024-11-09T00:21:53,551 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T00:21:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d794fbc6bc834420856e1d5f3f650c0e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e 2024-11-09T00:21:53,561 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T00:21:53,565 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9e3137292ff5c555eac85d85b3bc65c6 in 1022ms, sequenceid=358, compaction requested=false 2024-11-09T00:21:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:53,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T00:21:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T00:21:53,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T00:21:53,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-11-09T00:21:53,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6740 sec 2024-11-09T00:21:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:53,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:53,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T00:21:53,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e1ba20992cd24bb9ba57d0c8d2c09d35 is 50, key is test_row_0/A:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:53,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741912_1088 (size=14741) 2024-11-09T00:21:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111773865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111773885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111773894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111773899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111773908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111773986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T00:21:54,055 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:54,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T00:21:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:54,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:54,060 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:54,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111774204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e1ba20992cd24bb9ba57d0c8d2c09d35 2024-11-09T00:21:54,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T00:21:54,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:54,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:54,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:54,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:54,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:54,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/3ab9889bde7b411eaa6261a500e06b50 is 50, key is test_row_0/B:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741913_1089 (size=12301) 2024-11-09T00:21:54,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/3ab9889bde7b411eaa6261a500e06b50 2024-11-09T00:21:54,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/2fd07bced73d429dafcc7e95fc3b4d54 is 50, key is test_row_0/C:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:54,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741914_1090 (size=12301) 2024-11-09T00:21:54,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/2fd07bced73d429dafcc7e95fc3b4d54 2024-11-09T00:21:54,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e1ba20992cd24bb9ba57d0c8d2c09d35 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35 2024-11-09T00:21:54,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35, entries=200, sequenceid=376, filesize=14.4 K 2024-11-09T00:21:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/3ab9889bde7b411eaa6261a500e06b50 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50 2024-11-09T00:21:54,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50, entries=150, sequenceid=376, filesize=12.0 K 2024-11-09T00:21:54,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/2fd07bced73d429dafcc7e95fc3b4d54 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54 2024-11-09T00:21:54,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54, entries=150, sequenceid=376, filesize=12.0 K 2024-11-09T00:21:54,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9e3137292ff5c555eac85d85b3bc65c6 in 583ms, sequenceid=376, compaction requested=true 2024-11-09T00:21:54,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:54,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:54,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:54,359 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:54,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:54,360 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:54,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:54,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:54,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:54,361 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:54,361 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:54,361 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:54,361 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c07c908e6e584f4ab9f0dd981b175d8f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.8 K 2024-11-09T00:21:54,361 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:54,362 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:54,362 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:54,362 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a35a05a0601f4d8386e12fd60bbecbc5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=39.2 K 2024-11-09T00:21:54,362 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c07c908e6e584f4ab9f0dd981b175d8f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:54,362 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 950947bd47884fcdafe054577e1b6648, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731111711571 2024-11-09T00:21:54,362 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35a05a0601f4d8386e12fd60bbecbc5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:54,363 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e74637c59104ce29f69783c20d782cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731111711571 2024-11-09T00:21:54,363 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab9889bde7b411eaa6261a500e06b50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712744 2024-11-09T00:21:54,363 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ba20992cd24bb9ba57d0c8d2c09d35, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712738 2024-11-09T00:21:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:54,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T00:21:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:54,677 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T00:21:54,681 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#78 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:54,682 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/5d3d226869db48988b455fdcd5288575 is 50, key is test_row_0/B:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:54,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:54,694 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:54,695 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/90c7a14b05fb41bd8f4593a9da7a46f8 is 50, key is test_row_0/A:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/41fc28f096fe440a8f5e5060a2aa342b is 50, key is test_row_0/A:col10/1731111713862/Put/seqid=0 2024-11-09T00:21:54,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741915_1091 (size=13187) 2024-11-09T00:21:54,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741916_1092 (size=13187) 2024-11-09T00:21:54,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741917_1093 (size=12301) 2024-11-09T00:21:54,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111774787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111774895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111774895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,897 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111774904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,909 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111774915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,916 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111774924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:54,930 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 9e3137292ff5c555eac85d85b3bc65c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:21:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:55,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111775105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:55,134 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/5d3d226869db48988b455fdcd5288575 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/5d3d226869db48988b455fdcd5288575 2024-11-09T00:21:55,141 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 5d3d226869db48988b455fdcd5288575(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:55,141 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:55,141 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111714359; duration=0sec 2024-11-09T00:21:55,142 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:55,142 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:55,142 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:55,144 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:55,144 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:55,144 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:55,144 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eb49ea191a564b5086e42685f42fdc3d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=36.8 K 2024-11-09T00:21:55,145 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting eb49ea191a564b5086e42685f42fdc3d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731111710895 2024-11-09T00:21:55,146 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d794fbc6bc834420856e1d5f3f650c0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731111711571 2024-11-09T00:21:55,147 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd07bced73d429dafcc7e95fc3b4d54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712744 2024-11-09T00:21:55,159 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/90c7a14b05fb41bd8f4593a9da7a46f8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/90c7a14b05fb41bd8f4593a9da7a46f8 2024-11-09T00:21:55,163 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:55,164 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d6b0a8d21889444a8eb9496dd8fbe82a is 50, key is test_row_0/C:col10/1731111712744/Put/seqid=0 2024-11-09T00:21:55,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/41fc28f096fe440a8f5e5060a2aa342b 2024-11-09T00:21:55,177 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into 90c7a14b05fb41bd8f4593a9da7a46f8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:55,177 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:55,177 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111714359; duration=0sec 2024-11-09T00:21:55,177 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:55,178 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:55,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741918_1094 (size=13187) 2024-11-09T00:21:55,188 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/d6b0a8d21889444a8eb9496dd8fbe82a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d6b0a8d21889444a8eb9496dd8fbe82a 2024-11-09T00:21:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e738c5cb428b4293bb999321453a1472 is 50, key is test_row_0/B:col10/1731111713862/Put/seqid=0 2024-11-09T00:21:55,200 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into d6b0a8d21889444a8eb9496dd8fbe82a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:55,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:55,200 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111714360; duration=0sec 2024-11-09T00:21:55,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:55,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:55,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741919_1095 (size=12301) 2024-11-09T00:21:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111775414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:55,608 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e738c5cb428b4293bb999321453a1472 2024-11-09T00:21:55,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/10e1760d8d2b48a99c61d9cda445a084 is 50, key is test_row_0/C:col10/1731111713862/Put/seqid=0 2024-11-09T00:21:55,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741920_1096 (size=12301) 2024-11-09T00:21:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111775928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:56,030 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/10e1760d8d2b48a99c61d9cda445a084 2024-11-09T00:21:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/41fc28f096fe440a8f5e5060a2aa342b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b 2024-11-09T00:21:56,045 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b, entries=150, sequenceid=397, filesize=12.0 K 2024-11-09T00:21:56,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e738c5cb428b4293bb999321453a1472 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472 2024-11-09T00:21:56,052 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472, entries=150, sequenceid=397, filesize=12.0 K 2024-11-09T00:21:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/10e1760d8d2b48a99c61d9cda445a084 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084 2024-11-09T00:21:56,065 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084, entries=150, sequenceid=397, filesize=12.0 K 2024-11-09T00:21:56,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9e3137292ff5c555eac85d85b3bc65c6 in 1389ms, sequenceid=397, compaction requested=false 2024-11-09T00:21:56,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:56,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:56,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T00:21:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T00:21:56,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T00:21:56,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0080 sec 2024-11-09T00:21:56,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.0160 sec 2024-11-09T00:21:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T00:21:56,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:56,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T00:21:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T00:21:56,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:56,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:56,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T00:21:56,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T00:21:56,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:56,670 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T00:21:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/82e7ed408f8b4421aef2ace4b4f25101 is 50, key is test_row_0/A:col10/1731111714760/Put/seqid=0 2024-11-09T00:21:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741921_1097 (size=12301) 2024-11-09T00:21:56,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/82e7ed408f8b4421aef2ace4b4f25101 2024-11-09T00:21:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/c0148e5088a34664b888fb2c6472f6b5 is 50, key is test_row_0/B:col10/1731111714760/Put/seqid=0 2024-11-09T00:21:56,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741922_1098 (size=12301) 2024-11-09T00:21:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T00:21:56,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111776961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111776965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111776966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111776966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111776967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:56,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111777074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111777075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111777075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111777075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111777075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,125 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/c0148e5088a34664b888fb2c6472f6b5 2024-11-09T00:21:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T00:21:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/8dbb49384d56421f8699faa779074c48 is 50, key is test_row_0/C:col10/1731111714760/Put/seqid=0 2024-11-09T00:21:57,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741923_1099 (size=12301) 2024-11-09T00:21:57,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/8dbb49384d56421f8699faa779074c48 2024-11-09T00:21:57,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/82e7ed408f8b4421aef2ace4b4f25101 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101 2024-11-09T00:21:57,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101, entries=150, sequenceid=415, filesize=12.0 K 2024-11-09T00:21:57,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/c0148e5088a34664b888fb2c6472f6b5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5 2024-11-09T00:21:57,167 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5, entries=150, sequenceid=415, filesize=12.0 K 2024-11-09T00:21:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/8dbb49384d56421f8699faa779074c48 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48 2024-11-09T00:21:57,205 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48, entries=150, sequenceid=415, filesize=12.0 K 2024-11-09T00:21:57,207 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 9e3137292ff5c555eac85d85b3bc65c6 in 536ms, sequenceid=415, compaction requested=true 2024-11-09T00:21:57,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:57,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:57,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T00:21:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T00:21:57,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T00:21:57,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 692 msec 2024-11-09T00:21:57,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 707 msec 2024-11-09T00:21:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:57,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:57,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:21:57,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/8b8fdf8800ca4dc7a498f378d46974e3 is 50, key is test_row_0/A:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111777315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111777315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111777315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111777317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111777318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741924_1100 (size=17181) 2024-11-09T00:21:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111777436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111777437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111777438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111777439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111777441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T00:21:57,645 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:57,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111777645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111777645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-09T00:21:57,650 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T00:21:57,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:57,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111777654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111777654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111777654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/8b8fdf8800ca4dc7a498f378d46974e3 2024-11-09T00:21:57,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d7cfc1a162a149babf17b7fb13e887f6 is 50, key is test_row_0/B:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T00:21:57,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741925_1101 (size=12301) 2024-11-09T00:21:57,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d7cfc1a162a149babf17b7fb13e887f6 2024-11-09T00:21:57,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/eadba38f48154342b4b626b84dffc67a is 50, key is test_row_0/C:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:57,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741926_1102 (size=12301) 2024-11-09T00:21:57,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T00:21:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:57,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111777954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111777955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T00:21:57,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111777964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111777964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:57,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T00:21:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:57,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111777975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T00:21:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/eadba38f48154342b4b626b84dffc67a 2024-11-09T00:21:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/8b8fdf8800ca4dc7a498f378d46974e3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3 2024-11-09T00:21:58,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3, entries=250, sequenceid=439, filesize=16.8 K 2024-11-09T00:21:58,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/d7cfc1a162a149babf17b7fb13e887f6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6 2024-11-09T00:21:58,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6, entries=150, sequenceid=439, filesize=12.0 K 2024-11-09T00:21:58,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/eadba38f48154342b4b626b84dffc67a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a 2024-11-09T00:21:58,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a, entries=150, sequenceid=439, filesize=12.0 K 2024-11-09T00:21:58,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9e3137292ff5c555eac85d85b3bc65c6 in 970ms, sequenceid=439, compaction requested=true 2024-11-09T00:21:58,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:58,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:58,256 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:58,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:58,256 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:58,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:58,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:58,259 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:58,259 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:58,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:58,259 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,259 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,259 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/5d3d226869db48988b455fdcd5288575, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=48.9 K 2024-11-09T00:21:58,259 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/90c7a14b05fb41bd8f4593a9da7a46f8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=53.7 K 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c7a14b05fb41bd8f4593a9da7a46f8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712744 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3d226869db48988b455fdcd5288575, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712744 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41fc28f096fe440a8f5e5060a2aa342b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731111713837 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e738c5cb428b4293bb999321453a1472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731111713837 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c0148e5088a34664b888fb2c6472f6b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731111714745 2024-11-09T00:21:58,260 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e7ed408f8b4421aef2ace4b4f25101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731111714745 2024-11-09T00:21:58,261 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cfc1a162a149babf17b7fb13e887f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716964 2024-11-09T00:21:58,261 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8fdf8800ca4dc7a498f378d46974e3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716960 2024-11-09T00:21:58,272 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:58,273 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cb24a5ae3e6d40f2b72be8b799708164 is 50, key is test_row_0/B:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T00:21:58,277 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#91 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:58,277 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e5a0fdafadcc433db066830f9caa01e6 is 50, key is test_row_0/A:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:58,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T00:21:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,285 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:21:58,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741927_1103 (size=13323) 2024-11-09T00:21:58,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/fb31a898cefd4b1386fd9f34343570af is 50, key is test_row_0/A:col10/1731111717317/Put/seqid=0 2024-11-09T00:21:58,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741928_1104 (size=13323) 2024-11-09T00:21:58,306 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/cb24a5ae3e6d40f2b72be8b799708164 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cb24a5ae3e6d40f2b72be8b799708164 2024-11-09T00:21:58,323 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into cb24a5ae3e6d40f2b72be8b799708164(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:58,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:58,323 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=12, startTime=1731111718256; duration=0sec 2024-11-09T00:21:58,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:58,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:58,324 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:21:58,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741929_1105 (size=12301) 2024-11-09T00:21:58,333 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:21:58,333 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:58,333 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,334 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d6b0a8d21889444a8eb9496dd8fbe82a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=48.9 K 2024-11-09T00:21:58,334 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/fb31a898cefd4b1386fd9f34343570af 2024-11-09T00:21:58,334 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b0a8d21889444a8eb9496dd8fbe82a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731111712744 2024-11-09T00:21:58,335 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e1760d8d2b48a99c61d9cda445a084, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731111713837 2024-11-09T00:21:58,336 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbb49384d56421f8699faa779074c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731111714745 2024-11-09T00:21:58,337 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting eadba38f48154342b4b626b84dffc67a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716964 2024-11-09T00:21:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e72325c411fc4dfe9a04a6e30c03cd6c is 50, key is test_row_0/B:col10/1731111717317/Put/seqid=0 2024-11-09T00:21:58,375 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:58,377 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9 is 50, key is test_row_0/C:col10/1731111717286/Put/seqid=0 2024-11-09T00:21:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741930_1106 (size=12301) 2024-11-09T00:21:58,386 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e72325c411fc4dfe9a04a6e30c03cd6c 2024-11-09T00:21:58,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741931_1107 (size=13323) 2024-11-09T00:21:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a6a030221daa41d38d54d8e212c7cd77 is 50, key is test_row_0/C:col10/1731111717317/Put/seqid=0 2024-11-09T00:21:58,402 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9 2024-11-09T00:21:58,409 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into a42be97ac7bc47ee84e5d2ce4b2c1cc9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:58,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:58,410 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=12, startTime=1731111718257; duration=0sec 2024-11-09T00:21:58,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:58,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:21:58,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741932_1108 (size=12301) 2024-11-09T00:21:58,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a6a030221daa41d38d54d8e212c7cd77 2024-11-09T00:21:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/fb31a898cefd4b1386fd9f34343570af as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af 2024-11-09T00:21:58,430 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af, entries=150, sequenceid=451, filesize=12.0 K 2024-11-09T00:21:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e72325c411fc4dfe9a04a6e30c03cd6c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c 2024-11-09T00:21:58,440 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c, entries=150, sequenceid=451, filesize=12.0 K 2024-11-09T00:21:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/a6a030221daa41d38d54d8e212c7cd77 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77 2024-11-09T00:21:58,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77, entries=150, sequenceid=451, filesize=12.0 K 2024-11-09T00:21:58,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 9e3137292ff5c555eac85d85b3bc65c6 in 171ms, sequenceid=451, compaction requested=false 2024-11-09T00:21:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-09T00:21:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-09T00:21:58,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T00:21:58,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-11-09T00:21:58,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 819 msec 2024-11-09T00:21:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:58,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:21:58,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:21:58,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/5453a43713874697b1123a43c80b0abb is 50, key is test_row_0/A:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741933_1109 (size=17181) 2024-11-09T00:21:58,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/5453a43713874697b1123a43c80b0abb 2024-11-09T00:21:58,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/fd054a12136843049fd0a968b26583cf is 50, key is test_row_0/B:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:58,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741934_1110 (size=12301) 2024-11-09T00:21:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111778551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111778551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111778552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111778553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111778553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111778665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111778665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111778665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111778665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111778666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,705 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/e5a0fdafadcc433db066830f9caa01e6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e5a0fdafadcc433db066830f9caa01e6 2024-11-09T00:21:58,712 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into e5a0fdafadcc433db066830f9caa01e6(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:58,712 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:58,712 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=12, startTime=1731111718256; duration=0sec 2024-11-09T00:21:58,712 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:58,712 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T00:21:58,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:21:58,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:21:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-09T00:21:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:21:58,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:21:58,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:21:58,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:21:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111778875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111778875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111778875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111778875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111778876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:21:58,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/fd054a12136843049fd0a968b26583cf 2024-11-09T00:21:58,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1f541dfedda14de683a1af663820540d is 50, key is test_row_0/C:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:58,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T00:21:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:58,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741935_1111 (size=12301) 2024-11-09T00:21:59,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T00:21:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:21:59,173 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T00:21:59,173 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,176 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T00:21:59,176 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,176 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T00:21:59,176 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,177 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T00:21:59,177 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,178 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,178 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,178 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,178 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,178 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,179 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,179 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,179 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51306 deadline: 1731111779184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1731111779185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51284 deadline: 1731111779185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51302 deadline: 1731111779185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51246 deadline: 1731111779185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:21:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:21:59,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T00:21:59,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:59,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:21:59,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1f541dfedda14de683a1af663820540d 2024-11-09T00:21:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/5453a43713874697b1123a43c80b0abb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb 2024-11-09T00:21:59,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb, entries=250, sequenceid=465, filesize=16.8 K 2024-11-09T00:21:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/fd054a12136843049fd0a968b26583cf as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf 2024-11-09T00:21:59,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf, entries=150, sequenceid=465, filesize=12.0 K 2024-11-09T00:21:59,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/1f541dfedda14de683a1af663820540d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d 2024-11-09T00:21:59,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d, entries=150, sequenceid=465, filesize=12.0 K 2024-11-09T00:21:59,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9e3137292ff5c555eac85d85b3bc65c6 in 890ms, sequenceid=465, compaction requested=true 2024-11-09T00:21:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:21:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:59,371 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:21:59,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:59,371 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e3137292ff5c555eac85d85b3bc65c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:21:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/B is initiating minor compaction (all files) 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/A is initiating minor compaction (all files) 2024-11-09T00:21:59,373 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/B in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,373 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/A in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,373 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cb24a5ae3e6d40f2b72be8b799708164, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=37.0 K 2024-11-09T00:21:59,373 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e5a0fdafadcc433db066830f9caa01e6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=41.8 K 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cb24a5ae3e6d40f2b72be8b799708164, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716964 2024-11-09T00:21:59,373 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a0fdafadcc433db066830f9caa01e6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716964 2024-11-09T00:21:59,374 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e72325c411fc4dfe9a04a6e30c03cd6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731111717288 2024-11-09T00:21:59,374 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb31a898cefd4b1386fd9f34343570af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731111717288 2024-11-09T00:21:59,374 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting fd054a12136843049fd0a968b26583cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731111718476 2024-11-09T00:21:59,374 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5453a43713874697b1123a43c80b0abb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731111718466 2024-11-09T00:21:59,384 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#B#compaction#99 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:59,384 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/85cd61e9f8d040a3a4bd8c3ce514aff5 is 50, key is test_row_0/B:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:59,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741936_1112 (size=13425) 2024-11-09T00:21:59,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T00:21:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:21:59,414 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#A#compaction#100 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:59,415 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/138e3e577216490c825bc5cefe124ec5 is 50, key is test_row_0/A:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:21:59,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d3736bfcb61143b9aa5c30d6a54fdcc6 is 50, key is test_row_0/A:col10/1731111718551/Put/seqid=0 2024-11-09T00:21:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741937_1113 (size=13425) 2024-11-09T00:21:59,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741938_1114 (size=12301) 2024-11-09T00:21:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:21:59,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. as already flushing 2024-11-09T00:21:59,697 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T00:21:59,697 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T00:21:59,697 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,697 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,697 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,697 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,697 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T00:21:59,697 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,697 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,697 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,697 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,698 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,699 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T00:21:59,699 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T00:21:59,699 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,699 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:21:59,699 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,699 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,699 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,700 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:21:59,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:21:59,801 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/85cd61e9f8d040a3a4bd8c3ce514aff5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/85cd61e9f8d040a3a4bd8c3ce514aff5 2024-11-09T00:21:59,808 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/B of 9e3137292ff5c555eac85d85b3bc65c6 into 85cd61e9f8d040a3a4bd8c3ce514aff5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:59,808 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:59,808 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/B, priority=13, startTime=1731111719371; duration=0sec 2024-11-09T00:21:59,809 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:21:59,809 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:B 2024-11-09T00:21:59,809 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:21:59,811 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:21:59,811 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 9e3137292ff5c555eac85d85b3bc65c6/C is initiating minor compaction (all files) 2024-11-09T00:21:59,811 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e3137292ff5c555eac85d85b3bc65c6/C in TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:21:59,811 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp, totalSize=37.0 K 2024-11-09T00:21:59,812 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a42be97ac7bc47ee84e5d2ce4b2c1cc9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731111716964 2024-11-09T00:21:59,812 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a030221daa41d38d54d8e212c7cd77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731111717288 2024-11-09T00:21:59,813 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f541dfedda14de683a1af663820540d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731111718476 2024-11-09T00:21:59,824 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e3137292ff5c555eac85d85b3bc65c6#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:21:59,825 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ce31c661e1f044a697b230f34c925cee is 50, key is test_row_0/C:col10/1731111718480/Put/seqid=0 2024-11-09T00:21:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741939_1115 (size=13425) 2024-11-09T00:21:59,837 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/138e3e577216490c825bc5cefe124ec5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/138e3e577216490c825bc5cefe124ec5 2024-11-09T00:21:59,842 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/A of 9e3137292ff5c555eac85d85b3bc65c6 into 138e3e577216490c825bc5cefe124ec5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:21:59,842 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:21:59,842 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/A, priority=13, startTime=1731111719371; duration=0sec 2024-11-09T00:21:59,842 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:21:59,842 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:A 2024-11-09T00:21:59,847 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d3736bfcb61143b9aa5c30d6a54fdcc6 2024-11-09T00:21:59,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/59b4d5db0a6d4db498859af163e3c59a is 50, key is test_row_0/B:col10/1731111718551/Put/seqid=0 2024-11-09T00:21:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741940_1116 (size=12301) 2024-11-09T00:21:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:22:00,118 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:22:00,239 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/ce31c661e1f044a697b230f34c925cee as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ce31c661e1f044a697b230f34c925cee 2024-11-09T00:22:00,245 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e3137292ff5c555eac85d85b3bc65c6/C of 9e3137292ff5c555eac85d85b3bc65c6 into ce31c661e1f044a697b230f34c925cee(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:00,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:22:00,245 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., storeName=9e3137292ff5c555eac85d85b3bc65c6/C, priority=13, startTime=1731111719372; duration=0sec 2024-11-09T00:22:00,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:00,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e3137292ff5c555eac85d85b3bc65c6:C 2024-11-09T00:22:00,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/59b4d5db0a6d4db498859af163e3c59a 2024-11-09T00:22:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/15d4274647bc434d93bdceb433afd2a1 is 50, key is test_row_0/C:col10/1731111718551/Put/seqid=0 2024-11-09T00:22:00,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741941_1117 (size=12301) 2024-11-09T00:22:00,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/15d4274647bc434d93bdceb433afd2a1 2024-11-09T00:22:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/d3736bfcb61143b9aa5c30d6a54fdcc6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d3736bfcb61143b9aa5c30d6a54fdcc6 2024-11-09T00:22:00,693 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d3736bfcb61143b9aa5c30d6a54fdcc6, entries=150, sequenceid=490, filesize=12.0 K 2024-11-09T00:22:00,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/59b4d5db0a6d4db498859af163e3c59a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/59b4d5db0a6d4db498859af163e3c59a 2024-11-09T00:22:00,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/59b4d5db0a6d4db498859af163e3c59a, entries=150, sequenceid=490, filesize=12.0 K 2024-11-09T00:22:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/15d4274647bc434d93bdceb433afd2a1 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/15d4274647bc434d93bdceb433afd2a1 2024-11-09T00:22:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/15d4274647bc434d93bdceb433afd2a1, entries=150, sequenceid=490, filesize=12.0 K 2024-11-09T00:22:00,707 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for 9e3137292ff5c555eac85d85b3bc65c6 in 1299ms, sequenceid=490, compaction requested=false 2024-11-09T00:22:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9e3137292ff5c555eac85d85b3bc65c6: 2024-11-09T00:22:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:22:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-09T00:22:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-09T00:22:00,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T00:22:00,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9210 sec 2024-11-09T00:22:00,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.9260 sec 2024-11-09T00:22:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T00:22:00,945 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3082 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3225 2024-11-09T00:22:00,945 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:22:00,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1406 2024-11-09T00:22:00,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4217 rows 2024-11-09T00:22:00,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1415 2024-11-09T00:22:00,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4235 rows 2024-11-09T00:22:00,946 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:22:00,946 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:22:00,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:00,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:00,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:00,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:22:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:00,959 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111720959"}]},"ts":"1731111720959"} 2024-11-09T00:22:00,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:22:00,961 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:22:00,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:22:00,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, UNASSIGN}] 2024-11-09T00:22:00,969 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, UNASSIGN 2024-11-09T00:22:00,970 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9e3137292ff5c555eac85d85b3bc65c6, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:00,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, UNASSIGN because future has completed 2024-11-09T00:22:00,973 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:22:00,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:01,129 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:22:01,129 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:22:01,129 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 9e3137292ff5c555eac85d85b3bc65c6, disabling compactions & flushes 2024-11-09T00:22:01,129 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:22:01,129 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:22:01,129 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. after waiting 0 ms 2024-11-09T00:22:01,130 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:22:01,130 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 9e3137292ff5c555eac85d85b3bc65c6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T00:22:01,134 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/0cfe6e419ba647f495868cea4181f789 is 50, key is test_row_1/A:col10/1731111719697/Put/seqid=0 2024-11-09T00:22:01,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741942_1118 (size=9857) 2024-11-09T00:22:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:01,541 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/0cfe6e419ba647f495868cea4181f789 2024-11-09T00:22:01,549 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e77cb48e584d4260bbcfb038b4bbdb8c is 50, key is test_row_1/B:col10/1731111719697/Put/seqid=0 2024-11-09T00:22:01,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741943_1119 (size=9857) 2024-11-09T00:22:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:01,954 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e77cb48e584d4260bbcfb038b4bbdb8c 2024-11-09T00:22:01,963 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/98bf254d7df1489b9e2a634b831be095 is 50, key is test_row_1/C:col10/1731111719697/Put/seqid=0 2024-11-09T00:22:01,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741944_1120 (size=9857) 2024-11-09T00:22:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:02,368 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/98bf254d7df1489b9e2a634b831be095 2024-11-09T00:22:02,374 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/A/0cfe6e419ba647f495868cea4181f789 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/0cfe6e419ba647f495868cea4181f789 2024-11-09T00:22:02,379 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/0cfe6e419ba647f495868cea4181f789, entries=100, sequenceid=501, filesize=9.6 K 2024-11-09T00:22:02,380 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/B/e77cb48e584d4260bbcfb038b4bbdb8c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e77cb48e584d4260bbcfb038b4bbdb8c 2024-11-09T00:22:02,386 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e77cb48e584d4260bbcfb038b4bbdb8c, entries=100, sequenceid=501, filesize=9.6 K 2024-11-09T00:22:02,387 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/.tmp/C/98bf254d7df1489b9e2a634b831be095 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/98bf254d7df1489b9e2a634b831be095 2024-11-09T00:22:02,393 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/98bf254d7df1489b9e2a634b831be095, entries=100, sequenceid=501, filesize=9.6 K 2024-11-09T00:22:02,394 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9e3137292ff5c555eac85d85b3bc65c6 in 1264ms, sequenceid=501, compaction requested=true 2024-11-09T00:22:02,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/59288a558e304478b94e4b349e4119af, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/058863c750c3413c9128d1911eb7ab17, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/35e3ab41d8984b78b79c10b3b0dd4550, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b29133ed6adf41feb3845e3f754aa631, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a1582ce1cc5940d2a9ade8d98e82f309, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b0ae80b5c4ef4ea5b95f926093947e76, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a35a05a0601f4d8386e12fd60bbecbc5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/90c7a14b05fb41bd8f4593a9da7a46f8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e5a0fdafadcc433db066830f9caa01e6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb] to archive 2024-11-09T00:22:02,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:02,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/f164a3c775b8451381315d1a934ed0f6 2024-11-09T00:22:02,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/99787711ffa14549b592c9a8e3b29295 2024-11-09T00:22:02,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d7abe171f6034a478bfdc2e3d78ff09d 2024-11-09T00:22:02,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/ecf41c1769b942da8e17fab4ef23d010 2024-11-09T00:22:02,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/59288a558e304478b94e4b349e4119af to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/59288a558e304478b94e4b349e4119af 2024-11-09T00:22:02,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/09b7ad5bfce34495849232614586beb7 2024-11-09T00:22:02,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/058863c750c3413c9128d1911eb7ab17 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/058863c750c3413c9128d1911eb7ab17 2024-11-09T00:22:02,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6b92f5857aec404da6a635f2e8024895 2024-11-09T00:22:02,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/addf6c05217d42e8a4fa1684c49d4b58 2024-11-09T00:22:02,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/7395e276c01643eabbe4e6b6baa07afe 2024-11-09T00:22:02,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/35e3ab41d8984b78b79c10b3b0dd4550 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/35e3ab41d8984b78b79c10b3b0dd4550 2024-11-09T00:22:02,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e4e7b16e4e0d42c7beb2d4f6869a628f 2024-11-09T00:22:02,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/dfa4b04582ee44a7bbc11ff463f9f13d 2024-11-09T00:22:02,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/2ff6573eac9449cc984ffc412d91bc30 2024-11-09T00:22:02,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b29133ed6adf41feb3845e3f754aa631 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b29133ed6adf41feb3845e3f754aa631 2024-11-09T00:22:02,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/3508cba50b974214afd7d57a04ce17d9 2024-11-09T00:22:02,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a1582ce1cc5940d2a9ade8d98e82f309 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a1582ce1cc5940d2a9ade8d98e82f309 2024-11-09T00:22:02,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/debccfc7a9aa40c4a2834465e5e8ec8b 2024-11-09T00:22:02,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/afba5a349def4e05a5326053571a7525 2024-11-09T00:22:02,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b0ae80b5c4ef4ea5b95f926093947e76 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/b0ae80b5c4ef4ea5b95f926093947e76 2024-11-09T00:22:02,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e6b7f00a9001490886c0fa05a0d45051 2024-11-09T00:22:02,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/554ca2672cc944beac6029c2f95af735 2024-11-09T00:22:02,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6fe2e6d88327408c90d6ee6eb4272287 2024-11-09T00:22:02,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a35a05a0601f4d8386e12fd60bbecbc5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/a35a05a0601f4d8386e12fd60bbecbc5 2024-11-09T00:22:02,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/6e74637c59104ce29f69783c20d782cd 2024-11-09T00:22:02,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e1ba20992cd24bb9ba57d0c8d2c09d35 2024-11-09T00:22:02,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/90c7a14b05fb41bd8f4593a9da7a46f8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/90c7a14b05fb41bd8f4593a9da7a46f8 2024-11-09T00:22:02,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/41fc28f096fe440a8f5e5060a2aa342b 2024-11-09T00:22:02,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/82e7ed408f8b4421aef2ace4b4f25101 2024-11-09T00:22:02,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/8b8fdf8800ca4dc7a498f378d46974e3 2024-11-09T00:22:02,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e5a0fdafadcc433db066830f9caa01e6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/e5a0fdafadcc433db066830f9caa01e6 2024-11-09T00:22:02,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/fb31a898cefd4b1386fd9f34343570af 2024-11-09T00:22:02,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/5453a43713874697b1123a43c80b0abb 2024-11-09T00:22:02,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a7434e6646a34beaa7e89089b6624f88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/24221a5610204917a07bf476e032ff48, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2782badfa3124cdca7e5307fae687b71, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cd6480658634435a98128182fa7b148c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/7bcb07785e8b48909a9e1f0e53d579c3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0377977571e9453fb39d0574d548c09a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c07c908e6e584f4ab9f0dd981b175d8f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/5d3d226869db48988b455fdcd5288575, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cb24a5ae3e6d40f2b72be8b799708164, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf] to archive 2024-11-09T00:22:02,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:02,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a3101d4a02934558a4438dd1c440271e 2024-11-09T00:22:02,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/57265410573d4d568e41e6eb825fcab7 2024-11-09T00:22:02,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e637a5a4e07a41348ef5f1ee5642e059 2024-11-09T00:22:02,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a7434e6646a34beaa7e89089b6624f88 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a7434e6646a34beaa7e89089b6624f88 2024-11-09T00:22:02,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d8491b1296d143a38c7be54b1811d5fb 2024-11-09T00:22:02,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/680ffff3406f46f887599cbfc71c274e 2024-11-09T00:22:02,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/24221a5610204917a07bf476e032ff48 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/24221a5610204917a07bf476e032ff48 2024-11-09T00:22:02,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/227d3420dc0f4fc88a365711ca096a79 2024-11-09T00:22:02,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a531f96b2ca24b149a602f8396f3cef7 2024-11-09T00:22:02,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2abc9aebb61c485f9af1cf54bbcac214 2024-11-09T00:22:02,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2782badfa3124cdca7e5307fae687b71 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/2782badfa3124cdca7e5307fae687b71 2024-11-09T00:22:02,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0f20a1636d654b5ab9ce113a2a563fa3 2024-11-09T00:22:02,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/6b7d439afe444dc5963a1d93f220c315 2024-11-09T00:22:02,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cd6480658634435a98128182fa7b148c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cd6480658634435a98128182fa7b148c 2024-11-09T00:22:02,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/745e4ff8217e488eaad224b0fe5a918f 2024-11-09T00:22:02,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/67653ab66bc943549c0f03225c47d24f 2024-11-09T00:22:02,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/7bcb07785e8b48909a9e1f0e53d579c3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/7bcb07785e8b48909a9e1f0e53d579c3 2024-11-09T00:22:02,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/a8cfc574cc8442f2bc89673908d9aec8 2024-11-09T00:22:02,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cf2168c3ff964b1d8b3b95674097241a 2024-11-09T00:22:02,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0377977571e9453fb39d0574d548c09a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/0377977571e9453fb39d0574d548c09a 2024-11-09T00:22:02,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/22b364bc9fab492c8e99ba774f6efb29 2024-11-09T00:22:02,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d1cfbe972a21458da653d3de7369c6fc 2024-11-09T00:22:02,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c07c908e6e584f4ab9f0dd981b175d8f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c07c908e6e584f4ab9f0dd981b175d8f 2024-11-09T00:22:02,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/4972ee70b3c741d6b5a08e5b89d2475a 2024-11-09T00:22:02,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/950947bd47884fcdafe054577e1b6648 2024-11-09T00:22:02,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/5d3d226869db48988b455fdcd5288575 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/5d3d226869db48988b455fdcd5288575 2024-11-09T00:22:02,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/3ab9889bde7b411eaa6261a500e06b50 2024-11-09T00:22:02,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e738c5cb428b4293bb999321453a1472 2024-11-09T00:22:02,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/c0148e5088a34664b888fb2c6472f6b5 2024-11-09T00:22:02,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cb24a5ae3e6d40f2b72be8b799708164 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/cb24a5ae3e6d40f2b72be8b799708164 2024-11-09T00:22:02,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/d7cfc1a162a149babf17b7fb13e887f6 2024-11-09T00:22:02,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e72325c411fc4dfe9a04a6e30c03cd6c 2024-11-09T00:22:02,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/fd054a12136843049fd0a968b26583cf 2024-11-09T00:22:02,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0b7e5b1b8cf0476aa86906300d39c167, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/b1c80105e1a74127a5404fc8528fdfff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/446321c559ec41f5b25f524bb75c4bb8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a0f741e2263240be9462e2cd3333c74a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/97218660b26f4e91b37d79ae070b5cd0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/61172c25bac04ce6ac25e4d001b7edf6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eb49ea191a564b5086e42685f42fdc3d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d6b0a8d21889444a8eb9496dd8fbe82a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d] to archive 2024-11-09T00:22:02,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:02,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d5267864280e47fe8673b3f6ecd30134 2024-11-09T00:22:02,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/599e57b7ff1c4e7fa030ccf477ff5410 2024-11-09T00:22:02,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ec1113463397411997ed512ce03d63a7 2024-11-09T00:22:02,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0b7e5b1b8cf0476aa86906300d39c167 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0b7e5b1b8cf0476aa86906300d39c167 2024-11-09T00:22:02,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/9d96e733fc6a45329ee9b91290a8e47d 2024-11-09T00:22:02,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1c494f4905a64a739408db8a0b2395eb 2024-11-09T00:22:02,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/b1c80105e1a74127a5404fc8528fdfff to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/b1c80105e1a74127a5404fc8528fdfff 2024-11-09T00:22:02,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ed0c00766a584a0ea8b2a7ea2b414a10 2024-11-09T00:22:02,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/287494d0abdb40758e8e840a2a123c7d 2024-11-09T00:22:02,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a29d0b52a75b4a738c6ed1ec17e58341 2024-11-09T00:22:02,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/446321c559ec41f5b25f524bb75c4bb8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/446321c559ec41f5b25f524bb75c4bb8 2024-11-09T00:22:02,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/249326964fad4c118191d07b3b19ef49 2024-11-09T00:22:02,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a74850aae0d14526a0d269bd423214b6 2024-11-09T00:22:02,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a0f741e2263240be9462e2cd3333c74a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a0f741e2263240be9462e2cd3333c74a 2024-11-09T00:22:02,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/bb1abfaacfa541ec803cb5eea65a965c 2024-11-09T00:22:02,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/acf396ebea584cff826cba0b0f28a53e 2024-11-09T00:22:02,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/97218660b26f4e91b37d79ae070b5cd0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/97218660b26f4e91b37d79ae070b5cd0 2024-11-09T00:22:02,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/7dc1e4d29d50487c8241ad7055aeefc9 2024-11-09T00:22:02,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/c4f81f01652d4c0f855bd57ceb7878c0 2024-11-09T00:22:02,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/61172c25bac04ce6ac25e4d001b7edf6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/61172c25bac04ce6ac25e4d001b7edf6 2024-11-09T00:22:02,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/0f9ef39705f648dea1cd91c0fdc4ad8f 2024-11-09T00:22:02,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/f2af9b4eee534d1cb19160f6e246df88 2024-11-09T00:22:02,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eb49ea191a564b5086e42685f42fdc3d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eb49ea191a564b5086e42685f42fdc3d 2024-11-09T00:22:02,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/22274712de65451b888039a04fc7db61 2024-11-09T00:22:02,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d794fbc6bc834420856e1d5f3f650c0e 2024-11-09T00:22:02,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d6b0a8d21889444a8eb9496dd8fbe82a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/d6b0a8d21889444a8eb9496dd8fbe82a 2024-11-09T00:22:02,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/2fd07bced73d429dafcc7e95fc3b4d54 2024-11-09T00:22:02,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/10e1760d8d2b48a99c61d9cda445a084 2024-11-09T00:22:02,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/8dbb49384d56421f8699faa779074c48 2024-11-09T00:22:02,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a42be97ac7bc47ee84e5d2ce4b2c1cc9 2024-11-09T00:22:02,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/eadba38f48154342b4b626b84dffc67a 2024-11-09T00:22:02,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/a6a030221daa41d38d54d8e212c7cd77 2024-11-09T00:22:02,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/1f541dfedda14de683a1af663820540d 2024-11-09T00:22:02,556 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/recovered.edits/504.seqid, newMaxSeqId=504, maxSeqId=1 2024-11-09T00:22:02,557 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. 2024-11-09T00:22:02,557 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 9e3137292ff5c555eac85d85b3bc65c6: Waiting for close lock at 1731111721129Running coprocessor pre-close hooks at 1731111721129Disabling compacts and flushes for region at 1731111721129Disabling writes for close at 1731111721130 (+1 ms)Obtaining lock to block concurrent updates at 1731111721130Preparing flush snapshotting stores in 9e3137292ff5c555eac85d85b3bc65c6 at 1731111721130Finished memstore snapshotting TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731111721130Flushing stores of TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6. at 1731111721131 (+1 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/A: creating writer at 1731111721131Flushing 9e3137292ff5c555eac85d85b3bc65c6/A: appending metadata at 1731111721134 (+3 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/A: closing flushed file at 1731111721134Flushing 9e3137292ff5c555eac85d85b3bc65c6/B: creating writer at 1731111721546 (+412 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/B: appending metadata at 1731111721549 (+3 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/B: closing flushed file at 1731111721549Flushing 9e3137292ff5c555eac85d85b3bc65c6/C: creating writer at 1731111721960 (+411 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/C: appending metadata at 1731111721963 (+3 ms)Flushing 9e3137292ff5c555eac85d85b3bc65c6/C: closing flushed file at 1731111721963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3da777: reopening flushed file at 1731111722373 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1095890e: reopening flushed file at 1731111722380 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65113316: reopening flushed file at 1731111722386 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9e3137292ff5c555eac85d85b3bc65c6 in 1264ms, sequenceid=501, compaction requested=true at 1731111722394 (+8 ms)Writing region close event to WAL at 1731111722553 (+159 ms)Running coprocessor post-close hooks at 1731111722557 (+4 ms)Closed at 1731111722557 2024-11-09T00:22:02,560 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:22:02,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9e3137292ff5c555eac85d85b3bc65c6, regionState=CLOSED 2024-11-09T00:22:02,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:02,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T00:22:02,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e3137292ff5c555eac85d85b3bc65c6, server=ef986b38239e,35861,1731111692607 in 1.5900 sec 2024-11-09T00:22:02,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T00:22:02,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e3137292ff5c555eac85d85b3bc65c6, UNASSIGN in 1.5980 sec 2024-11-09T00:22:02,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T00:22:02,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6060 sec 2024-11-09T00:22:02,572 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111722572"}]},"ts":"1731111722572"} 2024-11-09T00:22:02,573 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:22:02,573 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:22:02,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6220 sec 2024-11-09T00:22:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T00:22:03,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:03,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:22:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,115 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T00:22:03,116 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,131 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:22:03,136 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/recovered.edits] 2024-11-09T00:22:03,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/0cfe6e419ba647f495868cea4181f789 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/0cfe6e419ba647f495868cea4181f789 2024-11-09T00:22:03,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/138e3e577216490c825bc5cefe124ec5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/138e3e577216490c825bc5cefe124ec5 2024-11-09T00:22:03,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d3736bfcb61143b9aa5c30d6a54fdcc6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/A/d3736bfcb61143b9aa5c30d6a54fdcc6 2024-11-09T00:22:03,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/59b4d5db0a6d4db498859af163e3c59a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/59b4d5db0a6d4db498859af163e3c59a 2024-11-09T00:22:03,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/85cd61e9f8d040a3a4bd8c3ce514aff5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/85cd61e9f8d040a3a4bd8c3ce514aff5 2024-11-09T00:22:03,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e77cb48e584d4260bbcfb038b4bbdb8c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/B/e77cb48e584d4260bbcfb038b4bbdb8c 2024-11-09T00:22:03,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/15d4274647bc434d93bdceb433afd2a1 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/15d4274647bc434d93bdceb433afd2a1 2024-11-09T00:22:03,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/98bf254d7df1489b9e2a634b831be095 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/98bf254d7df1489b9e2a634b831be095 2024-11-09T00:22:03,157 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ce31c661e1f044a697b230f34c925cee to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/C/ce31c661e1f044a697b230f34c925cee 2024-11-09T00:22:03,160 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/recovered.edits/504.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6/recovered.edits/504.seqid 2024-11-09T00:22:03,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/9e3137292ff5c555eac85d85b3bc65c6 2024-11-09T00:22:03,161 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:22:03,166 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T00:22:03,177 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:22:03,212 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:22:03,213 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,214 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:22:03,214 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111723214"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:03,219 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:22:03,219 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9e3137292ff5c555eac85d85b3bc65c6, NAME => 'TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:22:03,219 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:22:03,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111723219"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:03,222 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:22:03,223 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T00:22:03,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 114 msec 2024-11-09T00:22:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T00:22:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:22:03,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:03,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-488542015_22 at /127.0.0.1:35218 [Waiting for operation #344] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ef986b38239e:35861-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-60443270_22 at /127.0.0.1:35342 [Waiting for operation #330] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ef986b38239e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=640 (was 645), ProcessCount=11 (was 11), AvailableMemoryMB=3422 (was 4084) 2024-11-09T00:22:03,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=640, ProcessCount=11, AvailableMemoryMB=3421 2024-11-09T00:22:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:22:03,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:22:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:03,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:22:03,466 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:03,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-09T00:22:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:03,467 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:22:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741945_1121 (size=959) 2024-11-09T00:22:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:03,876 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:22:03,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741946_1122 (size=53) 2024-11-09T00:22:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:04,282 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:22:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c510ab9dfa980c00b700ca744b2587a4, disabling compactions & flushes 2024-11-09T00:22:04,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. after waiting 0 ms 2024-11-09T00:22:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c510ab9dfa980c00b700ca744b2587a4: Waiting for close lock at 1731111724283Disabling compacts and flushes for region at 1731111724283Disabling writes for close at 1731111724283Writing region close event to WAL at 1731111724283Closed at 1731111724283 2024-11-09T00:22:04,284 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:22:04,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111724284"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111724284"}]},"ts":"1731111724284"} 2024-11-09T00:22:04,287 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:22:04,289 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:22:04,289 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111724289"}]},"ts":"1731111724289"} 2024-11-09T00:22:04,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:22:04,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, ASSIGN}] 2024-11-09T00:22:04,293 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, ASSIGN 2024-11-09T00:22:04,293 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:22:04,444 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:04,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, ASSIGN because future has completed 2024-11-09T00:22:04,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:04,605 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,606 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:22:04,606 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,606 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:22:04,607 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,607 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,609 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,612 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName A 2024-11-09T00:22:04,612 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:04,613 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:04,613 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,615 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName B 2024-11-09T00:22:04,615 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:04,616 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:04,616 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,617 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName C 2024-11-09T00:22:04,617 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:04,617 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:04,618 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,618 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,619 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,619 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,620 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,620 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:22:04,621 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,623 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:22:04,623 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c510ab9dfa980c00b700ca744b2587a4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66821751, jitterRate=-0.004278317093849182}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:22:04,624 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:04,624 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c510ab9dfa980c00b700ca744b2587a4: Running coprocessor pre-open hook at 1731111724607Writing region info on filesystem at 1731111724607Initializing all the Stores at 1731111724608 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111724608Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111724609 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111724609Cleaning up temporary data from old regions at 1731111724620 (+11 ms)Running coprocessor post-open hooks at 1731111724624 (+4 ms)Region opened successfully at 1731111724624 2024-11-09T00:22:04,625 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., pid=32, masterSystemTime=1731111724600 2024-11-09T00:22:04,627 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,627 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:04,628 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:04,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:04,631 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38689 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=ef986b38239e,35861,1731111692607, table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T00:22:04,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T00:22:04,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 in 184 msec 2024-11-09T00:22:04,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T00:22:04,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, ASSIGN in 343 msec 2024-11-09T00:22:04,637 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:22:04,637 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111724637"}]},"ts":"1731111724637"} 2024-11-09T00:22:04,639 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:22:04,640 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:22:04,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-09T00:22:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-09T00:22:05,615 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:05,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:05,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:05,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:05,619 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcec947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:05,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9753ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:05,624 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:05,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:05,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:22:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:22:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@590288b4 2024-11-09T00:22:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:22:05,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:22:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:22:05,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:22:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:05,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741947_1123 (size=995) 2024-11-09T00:22:06,061 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T00:22:06,061 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T00:22:06,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:22:06,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE}] 2024-11-09T00:22:06,077 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE 2024-11-09T00:22:06,078 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE because future has completed 2024-11-09T00:22:06,080 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:22:06,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:06,233 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,233 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:22:06,234 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing c510ab9dfa980c00b700ca744b2587a4, disabling compactions & flushes 2024-11-09T00:22:06,234 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,234 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,234 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. after waiting 0 ms 2024-11-09T00:22:06,234 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,238 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T00:22:06,238 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,238 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for c510ab9dfa980c00b700ca744b2587a4: Waiting for close lock at 1731111726234Running coprocessor pre-close hooks at 1731111726234Disabling compacts and flushes for region at 1731111726234Disabling writes for close at 1731111726234Writing region close event to WAL at 1731111726234Running coprocessor post-close hooks at 1731111726238 (+4 ms)Closed at 1731111726238 2024-11-09T00:22:06,239 WARN [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: c510ab9dfa980c00b700ca744b2587a4 to self. 2024-11-09T00:22:06,241 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=CLOSED 2024-11-09T00:22:06,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:06,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-09T00:22:06,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 in 164 msec 2024-11-09T00:22:06,248 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE; state=CLOSED, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=true 2024-11-09T00:22:06,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE because future has completed 2024-11-09T00:22:06,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:06,557 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,557 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:22:06,557 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,557 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:22:06,557 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,558 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,562 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,567 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName A 2024-11-09T00:22:06,569 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:06,570 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:06,571 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,572 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName B 2024-11-09T00:22:06,572 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:06,573 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:06,573 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,574 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c510ab9dfa980c00b700ca744b2587a4 columnFamilyName C 2024-11-09T00:22:06,574 DEBUG [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:06,575 INFO [StoreOpener-c510ab9dfa980c00b700ca744b2587a4-1 {}] regionserver.HStore(327): Store=c510ab9dfa980c00b700ca744b2587a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:06,575 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,576 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,578 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,579 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,579 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,579 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:22:06,581 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,582 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c510ab9dfa980c00b700ca744b2587a4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75400596, jitterRate=0.12355643510818481}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:22:06,582 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,584 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c510ab9dfa980c00b700ca744b2587a4: Running coprocessor pre-open hook at 1731111726558Writing region info on filesystem at 1731111726558Initializing all the Stores at 1731111726559 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111726559Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111726559Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111726559Cleaning up temporary data from old regions at 1731111726579 (+20 ms)Running coprocessor post-open hooks at 1731111726582 (+3 ms)Region opened successfully at 1731111726584 (+2 ms) 2024-11-09T00:22:06,585 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., pid=37, masterSystemTime=1731111726553 2024-11-09T00:22:06,587 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,587 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,587 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=OPEN, openSeqNum=5, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:06,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-09T00:22:06,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 in 189 msec 2024-11-09T00:22:06,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T00:22:06,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, REOPEN/MOVE in 516 msec 2024-11-09T00:22:06,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T00:22:06,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-09T00:22:06,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-09T00:22:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-09T00:22:06,603 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:06,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52fc60cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9429d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4d15a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,614 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f3a66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63be82e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,618 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1c2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,620 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162bfbf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199484d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,623 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421e68d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,626 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c47260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c411a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68926b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dca7d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f81573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,635 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef014ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:06,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:06,637 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270c7997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:06,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:06,641 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,641 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,641 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,642 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,643 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,644 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,644 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,645 DEBUG [Thread-637 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,645 DEBUG [Thread-635 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,645 DEBUG [Thread-635 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,645 DEBUG [Thread-637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:06,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,647 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-09T00:22:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,651 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:06,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,652 DEBUG [Thread-626 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:06,653 DEBUG [Thread-626 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:06,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:06,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:22:06,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:06,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:22:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:06,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110998b98aa150484efd80142d1cee09f4cc_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111726665/Put/seqid=0 2024-11-09T00:22:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111786740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111786743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111786749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111786755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111786756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741948_1124 (size=14594) 2024-11-09T00:22:06,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:06,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111786865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111786875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111786875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111786875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111786875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:06,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:06,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111787085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111787085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111787085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111787085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111787086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:07,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:07,339 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110998b98aa150484efd80142d1cee09f4cc_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110998b98aa150484efd80142d1cee09f4cc_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:07,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/ed37b5b795b645adbc03ef54f079cf9d, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:07,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/ed37b5b795b645adbc03ef54f079cf9d is 175, key is test_row_0/A:col10/1731111726665/Put/seqid=0 2024-11-09T00:22:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111787395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111787395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111787395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111787395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111787405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741949_1125 (size=39549) 2024-11-09T00:22:07,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:07,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/ed37b5b795b645adbc03ef54f079cf9d 2024-11-09T00:22:07,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/359224811eb14264927fbba51b94ab46 is 50, key is test_row_0/B:col10/1731111726665/Put/seqid=0 2024-11-09T00:22:07,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741950_1126 (size=12001) 2024-11-09T00:22:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111787905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111787905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111787905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111787914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111787925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:07,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:07,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:08,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,119 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:22:08,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:08,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:08,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/359224811eb14264927fbba51b94ab46 2024-11-09T00:22:08,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b2dd2605ada7431d93b4809eff9c566d is 50, key is test_row_0/C:col10/1731111726665/Put/seqid=0 2024-11-09T00:22:08,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741951_1127 (size=12001) 2024-11-09T00:22:08,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b2dd2605ada7431d93b4809eff9c566d 2024-11-09T00:22:08,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/ed37b5b795b645adbc03ef54f079cf9d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d 2024-11-09T00:22:08,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d, entries=200, sequenceid=15, filesize=38.6 K 2024-11-09T00:22:08,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/359224811eb14264927fbba51b94ab46 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46 2024-11-09T00:22:08,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T00:22:08,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b2dd2605ada7431d93b4809eff9c566d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d 2024-11-09T00:22:08,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T00:22:08,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for c510ab9dfa980c00b700ca744b2587a4 in 1740ms, sequenceid=15, compaction requested=false 2024-11-09T00:22:08,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:08,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-09T00:22:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T00:22:08,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dbf35bf8ef8d4edf8bd60b24265e1033_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111726741/Put/seqid=0 2024-11-09T00:22:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741952_1128 (size=12154) 2024-11-09T00:22:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:08,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:08,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dbf35bf8ef8d4edf8bd60b24265e1033_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbf35bf8ef8d4edf8bd60b24265e1033_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/237e38bc97a7469290ab083f1d155064, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:08,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/237e38bc97a7469290ab083f1d155064 is 175, key is test_row_0/A:col10/1731111726741/Put/seqid=0 2024-11-09T00:22:08,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741953_1129 (size=30955) 2024-11-09T00:22:08,906 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/237e38bc97a7469290ab083f1d155064 2024-11-09T00:22:08,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/01dc86efd5684e28b6a07c2318a50820 is 50, key is test_row_0/B:col10/1731111726741/Put/seqid=0 2024-11-09T00:22:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:08,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111788937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111788938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:08,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111788941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:08,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111788943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:08,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741954_1130 (size=12001) 2024-11-09T00:22:08,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111788943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111789054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111789055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111789055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111789055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111789055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111789265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111789265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111789265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111789265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111789265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,352 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/01dc86efd5684e28b6a07c2318a50820 2024-11-09T00:22:09,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b7e290fdaced41a8ab5d339e4c477cad is 50, key is test_row_0/C:col10/1731111726741/Put/seqid=0 2024-11-09T00:22:09,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741955_1131 (size=12001) 2024-11-09T00:22:09,374 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b7e290fdaced41a8ab5d339e4c477cad 2024-11-09T00:22:09,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/237e38bc97a7469290ab083f1d155064 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064 2024-11-09T00:22:09,393 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T00:22:09,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/01dc86efd5684e28b6a07c2318a50820 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820 2024-11-09T00:22:09,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T00:22:09,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-09T00:22:09,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/b7e290fdaced41a8ab5d339e4c477cad as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad 2024-11-09T00:22:09,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T00:22:09,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for c510ab9dfa980c00b700ca744b2587a4 in 1010ms, sequenceid=41, compaction requested=false 2024-11-09T00:22:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-09T00:22:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-09T00:22:09,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T00:22:09,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7660 sec 2024-11-09T00:22:09,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.7890 sec 2024-11-09T00:22:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:09,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:09,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:09,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c948804b4c6d4f8894e7346f35b82622_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:09,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741956_1132 (size=19474) 2024-11-09T00:22:09,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111789615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111789616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111789618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111789623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111789625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c948804b4c6d4f8894e7346f35b82622_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c948804b4c6d4f8894e7346f35b82622_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:09,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/93f6ace979cb42fd8ca75ad718f110c5, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:09,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/93f6ace979cb42fd8ca75ad718f110c5 is 175, key is test_row_0/A:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:09,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741957_1133 (size=56733) 2024-11-09T00:22:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111789944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111789945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/93f6ace979cb42fd8ca75ad718f110c5 2024-11-09T00:22:10,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/464e8fadcd3c4e9ca8827a692431f59f is 50, key is test_row_0/B:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:10,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741958_1134 (size=12001) 2024-11-09T00:22:10,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/464e8fadcd3c4e9ca8827a692431f59f 2024-11-09T00:22:10,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cd739ea42e8f43d38fc20adb39a0edc2 is 50, key is test_row_0/C:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:10,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741959_1135 (size=12001) 2024-11-09T00:22:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111790255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cd739ea42e8f43d38fc20adb39a0edc2 2024-11-09T00:22:10,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/93f6ace979cb42fd8ca75ad718f110c5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5 2024-11-09T00:22:10,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5, entries=300, sequenceid=56, filesize=55.4 K 2024-11-09T00:22:10,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/464e8fadcd3c4e9ca8827a692431f59f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f 2024-11-09T00:22:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T00:22:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cd739ea42e8f43d38fc20adb39a0edc2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2 2024-11-09T00:22:10,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T00:22:10,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c510ab9dfa980c00b700ca744b2587a4 in 970ms, sequenceid=56, compaction requested=true 2024-11-09T00:22:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:10,549 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:10,549 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:10,551 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:10,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:10,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:10,551 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:10,551 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,551 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=35.2 K 2024-11-09T00:22:10,551 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,551 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=124.3 K 2024-11-09T00:22:10,551 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,552 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5] 2024-11-09T00:22:10,552 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 359224811eb14264927fbba51b94ab46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111726658 2024-11-09T00:22:10,552 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed37b5b795b645adbc03ef54f079cf9d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111726658 2024-11-09T00:22:10,553 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 01dc86efd5684e28b6a07c2318a50820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111726739 2024-11-09T00:22:10,553 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237e38bc97a7469290ab083f1d155064, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111726739 2024-11-09T00:22:10,554 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 464e8fadcd3c4e9ca8827a692431f59f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111729577 2024-11-09T00:22:10,555 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f6ace979cb42fd8ca75ad718f110c5, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111728933 2024-11-09T00:22:10,572 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#117 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:10,573 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/646a9ed2628742ebaaf513665e36a477 is 50, key is test_row_0/B:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:10,577 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:10,582 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ba054d64ef9f4d54b9d672659b14b8c3_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:10,593 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ba054d64ef9f4d54b9d672659b14b8c3_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:10,593 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ba054d64ef9f4d54b9d672659b14b8c3_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:10,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741960_1136 (size=12104) 2024-11-09T00:22:10,622 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/646a9ed2628742ebaaf513665e36a477 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/646a9ed2628742ebaaf513665e36a477 2024-11-09T00:22:10,645 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 646a9ed2628742ebaaf513665e36a477(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:10,645 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741961_1137 (size=4469) 2024-11-09T00:22:10,647 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=13, startTime=1731111730548; duration=0sec 2024-11-09T00:22:10,648 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:10,649 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:10,649 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:10,651 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#118 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:10,651 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/1e00cfc70dc44c43986f47cf0b970782 is 175, key is test_row_0/A:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:10,653 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:10,654 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/C is initiating minor compaction (all files) 2024-11-09T00:22:10,654 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/C in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,654 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=35.2 K 2024-11-09T00:22:10,655 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b2dd2605ada7431d93b4809eff9c566d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111726658 2024-11-09T00:22:10,656 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e290fdaced41a8ab5d339e4c477cad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111726739 2024-11-09T00:22:10,656 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cd739ea42e8f43d38fc20adb39a0edc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111729577 2024-11-09T00:22:10,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741962_1138 (size=31058) 2024-11-09T00:22:10,701 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/1e00cfc70dc44c43986f47cf0b970782 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782 2024-11-09T00:22:10,704 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:10,704 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/42d0a0f9c359447ea3462ee680f2048f is 50, key is test_row_0/C:col10/1731111729578/Put/seqid=0 2024-11-09T00:22:10,715 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into 1e00cfc70dc44c43986f47cf0b970782(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:10,715 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:10,716 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=13, startTime=1731111730548; duration=0sec 2024-11-09T00:22:10,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:10,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:10,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741963_1139 (size=12104) 2024-11-09T00:22:10,756 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/42d0a0f9c359447ea3462ee680f2048f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/42d0a0f9c359447ea3462ee680f2048f 2024-11-09T00:22:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:10,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:10,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T00:22:10,772 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/C of c510ab9dfa980c00b700ca744b2587a4 into 42d0a0f9c359447ea3462ee680f2048f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:10,772 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:10,772 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/C, priority=13, startTime=1731111730549; duration=0sec 2024-11-09T00:22:10,773 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:10,773 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:10,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c1a7540b8654d7789d6edc1b2c8ee2b_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111730766/Put/seqid=0 2024-11-09T00:22:10,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741964_1140 (size=12154) 2024-11-09T00:22:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-09T00:22:10,814 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:10,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-09T00:22:10,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:10,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:10,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:10,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111790787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111790789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111790821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111790831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111790831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:10,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111790925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111790925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111790945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111790945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111790945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:10,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:10,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:11,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111791136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111791144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111791154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111791154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111791154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:11,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c1a7540b8654d7789d6edc1b2c8ee2b_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c1a7540b8654d7789d6edc1b2c8ee2b_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:11,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/3a817781fc974c32b6f66d448183b2e3, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:11,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/3a817781fc974c32b6f66d448183b2e3 is 175, key is test_row_0/A:col10/1731111730766/Put/seqid=0 2024-11-09T00:22:11,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741965_1141 (size=30955) 2024-11-09T00:22:11,233 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/3a817781fc974c32b6f66d448183b2e3 2024-11-09T00:22:11,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/7b8b52d3b3bb474892ddee16b9bfc08e is 50, key is test_row_0/B:col10/1731111730766/Put/seqid=0 2024-11-09T00:22:11,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741966_1142 (size=12001) 2024-11-09T00:22:11,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:11,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111791445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111791455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111791465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111791465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111791465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:11,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/7b8b52d3b3bb474892ddee16b9bfc08e 2024-11-09T00:22:11,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/43113e7d71bb4613aea3fac84beea887 is 50, key is test_row_0/C:col10/1731111730766/Put/seqid=0 2024-11-09T00:22:11,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741967_1143 (size=12001) 2024-11-09T00:22:11,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/43113e7d71bb4613aea3fac84beea887 2024-11-09T00:22:11,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/3a817781fc974c32b6f66d448183b2e3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3 2024-11-09T00:22:11,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3, entries=150, sequenceid=83, filesize=30.2 K 2024-11-09T00:22:11,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/7b8b52d3b3bb474892ddee16b9bfc08e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e 2024-11-09T00:22:11,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T00:22:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/43113e7d71bb4613aea3fac84beea887 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887 2024-11-09T00:22:11,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T00:22:11,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for c510ab9dfa980c00b700ca744b2587a4 in 986ms, sequenceid=83, compaction requested=false 2024-11-09T00:22:11,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:11,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T00:22:11,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T00:22:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109be28a2ddd1f84a8185528b871a41942f_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111730794/Put/seqid=0 2024-11-09T00:22:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741968_1144 (size=12154) 2024-11-09T00:22:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:11,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111791991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111791998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111792000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111792001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111792003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111792104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111792114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111792114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111792114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111792114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111792314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111792324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111792324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111792325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111792325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:12,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109be28a2ddd1f84a8185528b871a41942f_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be28a2ddd1f84a8185528b871a41942f_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:12,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/870517740608460a868301c4a4aa0d8e, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:12,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/870517740608460a868301c4a4aa0d8e is 175, key is test_row_0/A:col10/1731111730794/Put/seqid=0 2024-11-09T00:22:12,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741969_1145 (size=30955) 2024-11-09T00:22:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111792624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111792635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111792635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111792635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111792635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:12,760 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/870517740608460a868301c4a4aa0d8e 2024-11-09T00:22:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/1c3542f38efb4dd1973effdef4e548a1 is 50, key is test_row_0/B:col10/1731111730794/Put/seqid=0 2024-11-09T00:22:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741970_1146 (size=12001) 2024-11-09T00:22:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111793134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:13,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111793144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111793144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111793145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:13,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111793155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:13,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/1c3542f38efb4dd1973effdef4e548a1 2024-11-09T00:22:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/21fb896544124020aa252fea563aeba0 is 50, key is test_row_0/C:col10/1731111730794/Put/seqid=0 2024-11-09T00:22:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741971_1147 (size=12001) 2024-11-09T00:22:13,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/21fb896544124020aa252fea563aeba0 2024-11-09T00:22:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/870517740608460a868301c4a4aa0d8e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e 2024-11-09T00:22:13,253 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e, entries=150, sequenceid=97, filesize=30.2 K 2024-11-09T00:22:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/1c3542f38efb4dd1973effdef4e548a1 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1 2024-11-09T00:22:13,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T00:22:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/21fb896544124020aa252fea563aeba0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0 2024-11-09T00:22:13,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T00:22:13,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for c510ab9dfa980c00b700ca744b2587a4 in 1376ms, sequenceid=97, compaction requested=true 2024-11-09T00:22:13,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:13,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-09T00:22:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-09T00:22:13,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T00:22:13,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-09T00:22:13,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.4790 sec 2024-11-09T00:22:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:14,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:14,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:22:14,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099e8d68451f574463a6d0380bcf589595_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:14,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741972_1148 (size=14594) 2024-11-09T00:22:14,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111794165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111794166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099e8d68451f574463a6d0380bcf589595_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099e8d68451f574463a6d0380bcf589595_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:14,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/9dd11da23c154413b9a057a329845425, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:14,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/9dd11da23c154413b9a057a329845425 is 175, key is test_row_0/A:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:14,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741973_1149 (size=39549) 2024-11-09T00:22:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111794168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111794169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111794170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111794275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111794275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111794284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111794284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111794294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111794494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111794494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111794494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111794494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111794505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/9dd11da23c154413b9a057a329845425 2024-11-09T00:22:14,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/bb6a55a784404648b6bc8f3d65c1101d is 50, key is test_row_0/B:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:14,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741974_1150 (size=12001) 2024-11-09T00:22:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111794805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111794805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111794805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111794814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111794814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T00:22:14,975 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:14,976 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T00:22:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:14,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:14,979 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:14,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:15,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/bb6a55a784404648b6bc8f3d65c1101d 2024-11-09T00:22:15,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/c3c385fb1fbd4d6db1bfd8b9d1448602 is 50, key is test_row_0/C:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:15,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741975_1151 (size=12001) 2024-11-09T00:22:15,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/c3c385fb1fbd4d6db1bfd8b9d1448602 2024-11-09T00:22:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/9dd11da23c154413b9a057a329845425 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425 2024-11-09T00:22:15,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425, entries=200, sequenceid=121, filesize=38.6 K 2024-11-09T00:22:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/bb6a55a784404648b6bc8f3d65c1101d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d 2024-11-09T00:22:15,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T00:22:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/c3c385fb1fbd4d6db1bfd8b9d1448602 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602 2024-11-09T00:22:15,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T00:22:15,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c510ab9dfa980c00b700ca744b2587a4 in 911ms, sequenceid=121, compaction requested=true 2024-11-09T00:22:15,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:15,060 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:15,060 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:15,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:15,061 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:15,061 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:15,062 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:15,062 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:15,062 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:15,062 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:15,062 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/646a9ed2628742ebaaf513665e36a477, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.0 K 2024-11-09T00:22:15,062 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=129.4 K 2024-11-09T00:22:15,062 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:15,062 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425] 2024-11-09T00:22:15,062 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 646a9ed2628742ebaaf513665e36a477, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111729577 2024-11-09T00:22:15,063 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8b52d3b3bb474892ddee16b9bfc08e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731111729619 2024-11-09T00:22:15,063 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e00cfc70dc44c43986f47cf0b970782, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111729577 2024-11-09T00:22:15,063 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3542f38efb4dd1973effdef4e548a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731111730786 2024-11-09T00:22:15,063 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a817781fc974c32b6f66d448183b2e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731111729619 2024-11-09T00:22:15,064 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting bb6a55a784404648b6bc8f3d65c1101d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731998 2024-11-09T00:22:15,064 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870517740608460a868301c4a4aa0d8e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731111730786 2024-11-09T00:22:15,064 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd11da23c154413b9a057a329845425, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731985 2024-11-09T00:22:15,074 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:15,075 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:15,076 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/922fdef4e11a4477b404097139513aac is 50, key is test_row_0/B:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:15,078 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411098f2a9070667140bcb9887d22142416d7_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:15,083 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411098f2a9070667140bcb9887d22142416d7_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:15,083 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098f2a9070667140bcb9887d22142416d7_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:15,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741976_1152 (size=12241) 2024-11-09T00:22:15,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741977_1153 (size=4469) 2024-11-09T00:22:15,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T00:22:15,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:15,132 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:15,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a004b83a432b4d80bba4b10592e23a78_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111734167/Put/seqid=0 2024-11-09T00:22:15,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741978_1154 (size=12254) 2024-11-09T00:22:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:15,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:15,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111795340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111795341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111795341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111795342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111795343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111795455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111795455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111795455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111795455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111795455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,489 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#130 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:15,490 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/803f88d16131479d9fa35729247d45e6 is 175, key is test_row_0/A:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:15,494 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/922fdef4e11a4477b404097139513aac as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/922fdef4e11a4477b404097139513aac 2024-11-09T00:22:15,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741979_1155 (size=31195) 2024-11-09T00:22:15,500 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 922fdef4e11a4477b404097139513aac(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:15,500 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=12, startTime=1731111735060; duration=0sec 2024-11-09T00:22:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:15,502 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:15,502 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/C is initiating minor compaction (all files) 2024-11-09T00:22:15,502 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/C in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:15,502 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/42d0a0f9c359447ea3462ee680f2048f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.0 K 2024-11-09T00:22:15,503 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d0a0f9c359447ea3462ee680f2048f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731111729577 2024-11-09T00:22:15,503 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 43113e7d71bb4613aea3fac84beea887, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731111729619 2024-11-09T00:22:15,503 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 21fb896544124020aa252fea563aeba0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731111730786 2024-11-09T00:22:15,505 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c385fb1fbd4d6db1bfd8b9d1448602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731998 2024-11-09T00:22:15,511 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/803f88d16131479d9fa35729247d45e6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6 2024-11-09T00:22:15,523 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into 803f88d16131479d9fa35729247d45e6(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:15,523 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:15,523 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=12, startTime=1731111735060; duration=0sec 2024-11-09T00:22:15,523 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:15,523 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:15,524 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#C#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:15,525 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/08cd592a7bbf4a71a39bdc384c498c2a is 50, key is test_row_0/C:col10/1731111731999/Put/seqid=0 2024-11-09T00:22:15,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741980_1156 (size=12241) 2024-11-09T00:22:15,542 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/08cd592a7bbf4a71a39bdc384c498c2a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/08cd592a7bbf4a71a39bdc384c498c2a 2024-11-09T00:22:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:15,552 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/C of c510ab9dfa980c00b700ca744b2587a4 into 08cd592a7bbf4a71a39bdc384c498c2a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:15,552 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:15,552 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/C, priority=12, startTime=1731111735060; duration=0sec 2024-11-09T00:22:15,553 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:15,553 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:15,558 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a004b83a432b4d80bba4b10592e23a78_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a004b83a432b4d80bba4b10592e23a78_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:15,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/7027e006b11d479fbfebb3fd18f74d6f, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/7027e006b11d479fbfebb3fd18f74d6f is 175, key is test_row_0/A:col10/1731111734167/Put/seqid=0 2024-11-09T00:22:15,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741981_1157 (size=31055) 2024-11-09T00:22:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111795664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111795664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111795665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111795665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111795665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111795974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111795974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111795974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,976 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/7027e006b11d479fbfebb3fd18f74d6f 2024-11-09T00:22:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111795974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111795975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/416c338a2af1432481468698485f28fb is 50, key is test_row_0/B:col10/1731111734167/Put/seqid=0 2024-11-09T00:22:15,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741982_1158 (size=12101) 2024-11-09T00:22:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:16,389 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/416c338a2af1432481468698485f28fb 2024-11-09T00:22:16,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0db9ebc475104e60a3a44ddcf2717e91 is 50, key is test_row_0/C:col10/1731111734167/Put/seqid=0 2024-11-09T00:22:16,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741983_1159 (size=12101) 2024-11-09T00:22:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111796484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111796484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111796485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111796485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111796485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:16,808 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0db9ebc475104e60a3a44ddcf2717e91 2024-11-09T00:22:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/7027e006b11d479fbfebb3fd18f74d6f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f 2024-11-09T00:22:16,820 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f, entries=150, sequenceid=134, filesize=30.3 K 2024-11-09T00:22:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/416c338a2af1432481468698485f28fb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb 2024-11-09T00:22:16,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb, entries=150, sequenceid=134, filesize=11.8 K 2024-11-09T00:22:16,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0db9ebc475104e60a3a44ddcf2717e91 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91 2024-11-09T00:22:16,834 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91, entries=150, sequenceid=134, filesize=11.8 K 2024-11-09T00:22:16,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c510ab9dfa980c00b700ca744b2587a4 in 1704ms, sequenceid=134, compaction requested=false 2024-11-09T00:22:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T00:22:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T00:22:16,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T00:22:16,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-09T00:22:16,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8650 sec 2024-11-09T00:22:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T00:22:17,125 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:17,126 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T00:22:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:17,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:17,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:17,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:17,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T00:22:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:17,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:22:17,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983940eec83a74be3bf8365c9bf64384f_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111735341/Put/seqid=0 2024-11-09T00:22:17,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741984_1160 (size=12304) 2024-11-09T00:22:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:17,447 INFO [master/ef986b38239e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T00:22:17,447 INFO [master/ef986b38239e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T00:22:17,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111797510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111797510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111797511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111797512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111797513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:17,728 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983940eec83a74be3bf8365c9bf64384f_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983940eec83a74be3bf8365c9bf64384f_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/333ed85247444dfca80fcb09d5743e88, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/333ed85247444dfca80fcb09d5743e88 is 175, key is test_row_0/A:col10/1731111735341/Put/seqid=0 2024-11-09T00:22:17,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741985_1161 (size=31105) 2024-11-09T00:22:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111797834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111797834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111797834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111797834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111797834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,136 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/333ed85247444dfca80fcb09d5743e88 2024-11-09T00:22:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4eb6fb7db5fa4b7189316807eb7b545d is 50, key is test_row_0/B:col10/1731111735341/Put/seqid=0 2024-11-09T00:22:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111798145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111798145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111798145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111798145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741986_1162 (size=12151) 2024-11-09T00:22:18,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111798145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4eb6fb7db5fa4b7189316807eb7b545d 2024-11-09T00:22:18,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cff2c16a061d4681b282ecfdb8463db0 is 50, key is test_row_0/C:col10/1731111735341/Put/seqid=0 2024-11-09T00:22:18,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741987_1163 (size=12151) 2024-11-09T00:22:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:18,570 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cff2c16a061d4681b282ecfdb8463db0 2024-11-09T00:22:18,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/333ed85247444dfca80fcb09d5743e88 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88 2024-11-09T00:22:18,581 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88, entries=150, sequenceid=160, filesize=30.4 K 2024-11-09T00:22:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4eb6fb7db5fa4b7189316807eb7b545d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d 2024-11-09T00:22:18,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T00:22:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/cff2c16a061d4681b282ecfdb8463db0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0 2024-11-09T00:22:18,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0, entries=150, sequenceid=160, filesize=11.9 K 2024-11-09T00:22:18,597 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c510ab9dfa980c00b700ca744b2587a4 in 1315ms, sequenceid=160, compaction requested=true 2024-11-09T00:22:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T00:22:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T00:22:18,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T00:22:18,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-09T00:22:18,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4760 sec 2024-11-09T00:22:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:18,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:18,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T00:22:18,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f83861613e96417c8a52e1e5c6c191d9_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:18,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741988_1164 (size=17284) 2024-11-09T00:22:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111798691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111798692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111798693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111798694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111798694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111798804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111798804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111798805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111798804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:18,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111798815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111799015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111799015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111799016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111799015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111799028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:19,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f83861613e96417c8a52e1e5c6c191d9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f83861613e96417c8a52e1e5c6c191d9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:19,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c3f7a58648f646ad8667723dfb20e6c1, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:19,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c3f7a58648f646ad8667723dfb20e6c1 is 175, key is test_row_0/A:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:19,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741989_1165 (size=48389) 2024-11-09T00:22:19,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c3f7a58648f646ad8667723dfb20e6c1 2024-11-09T00:22:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T00:22:19,285 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:19,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T00:22:19,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/318a52cbffcc4c0db42035cc3e9d57f6 is 50, key is test_row_0/B:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:19,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:19,297 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:19,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111799325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111799325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111799334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111799335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741990_1166 (size=12151) 2024-11-09T00:22:19,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/318a52cbffcc4c0db42035cc3e9d57f6 2024-11-09T00:22:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111799335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0dabbdf578a64d9caf8fe733e9285829 is 50, key is test_row_0/C:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:19,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741991_1167 (size=12151) 2024-11-09T00:22:19,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:19,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:19,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0dabbdf578a64d9caf8fe733e9285829 2024-11-09T00:22:19,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c3f7a58648f646ad8667723dfb20e6c1 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1 2024-11-09T00:22:19,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1, entries=250, sequenceid=175, filesize=47.3 K 2024-11-09T00:22:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/318a52cbffcc4c0db42035cc3e9d57f6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6 2024-11-09T00:22:19,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T00:22:19,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/0dabbdf578a64d9caf8fe733e9285829 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829 2024-11-09T00:22:19,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111799844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111799845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111799845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111799846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T00:22:19,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for c510ab9dfa980c00b700ca744b2587a4 in 1197ms, sequenceid=175, compaction requested=true 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:19,854 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:19,854 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:19,858 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141744 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:19,859 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:19,859 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,859 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=138.4 K 2024-11-09T00:22:19,859 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,859 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1] 2024-11-09T00:22:19,859 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:19,859 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:19,859 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,859 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/922fdef4e11a4477b404097139513aac, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.5 K 2024-11-09T00:22:19,860 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 922fdef4e11a4477b404097139513aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731998 2024-11-09T00:22:19,860 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803f88d16131479d9fa35729247d45e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731998 2024-11-09T00:22:19,860 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 416c338a2af1432481468698485f28fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731111734160 2024-11-09T00:22:19,861 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7027e006b11d479fbfebb3fd18f74d6f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731111734160 2024-11-09T00:22:19,861 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eb6fb7db5fa4b7189316807eb7b545d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731111735339 2024-11-09T00:22:19,861 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 333ed85247444dfca80fcb09d5743e88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731111735339 2024-11-09T00:22:19,862 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 318a52cbffcc4c0db42035cc3e9d57f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737510 2024-11-09T00:22:19,862 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f7a58648f646ad8667723dfb20e6c1, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737509 2024-11-09T00:22:19,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:19,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:22:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:19,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f65623e0cf9a4b55956d6a91ee4515f1_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111738692/Put/seqid=0 2024-11-09T00:22:19,910 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:19,911 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/3396c938709543249055a26fbd5c3705 is 50, key is test_row_0/B:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:19,913 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:19,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:19,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:19,945 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109a6b3648f7cac414a9db017881a85fc69_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:19,948 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109a6b3648f7cac414a9db017881a85fc69_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:19,949 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a6b3648f7cac414a9db017881a85fc69_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:19,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111799963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:19,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741992_1168 (size=14794) 2024-11-09T00:22:19,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741993_1169 (size=12527) 2024-11-09T00:22:20,003 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/3396c938709543249055a26fbd5c3705 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3396c938709543249055a26fbd5c3705 2024-11-09T00:22:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741994_1170 (size=4469) 2024-11-09T00:22:20,014 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 3396c938709543249055a26fbd5c3705(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:20,014 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:20,015 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=12, startTime=1731111739854; duration=0sec 2024-11-09T00:22:20,015 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:20,015 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:20,015 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:20,017 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:20,017 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/C is initiating minor compaction (all files) 2024-11-09T00:22:20,017 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/C in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,017 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/08cd592a7bbf4a71a39bdc384c498c2a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.5 K 2024-11-09T00:22:20,020 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 08cd592a7bbf4a71a39bdc384c498c2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731111731998 2024-11-09T00:22:20,021 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db9ebc475104e60a3a44ddcf2717e91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731111734160 2024-11-09T00:22:20,022 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#143 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:20,023 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/e77ed50bd2d940c5953b58fd6cd1553c is 175, key is test_row_0/A:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:20,025 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cff2c16a061d4681b282ecfdb8463db0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731111735339 2024-11-09T00:22:20,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dabbdf578a64d9caf8fe733e9285829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737510 2024-11-09T00:22:20,051 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:20,051 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/2fcd5c722ba444a5a22ac13d1fcf2fe5 is 50, key is test_row_0/C:col10/1731111738656/Put/seqid=0 2024-11-09T00:22:20,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741995_1171 (size=31481) 2024-11-09T00:22:20,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741996_1172 (size=12527) 2024-11-09T00:22:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111800074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,079 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/e77ed50bd2d940c5953b58fd6cd1553c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c 2024-11-09T00:22:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,088 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into e77ed50bd2d940c5953b58fd6cd1553c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:20,088 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:20,088 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=12, startTime=1731111739854; duration=0sec 2024-11-09T00:22:20,088 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:20,088 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:20,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111800285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:20,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f65623e0cf9a4b55956d6a91ee4515f1_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f65623e0cf9a4b55956d6a91ee4515f1_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:20,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/11fd507eb3414298b6ba7f81fcc5975d, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:20,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/11fd507eb3414298b6ba7f81fcc5975d is 175, key is test_row_0/A:col10/1731111738692/Put/seqid=0 2024-11-09T00:22:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741997_1173 (size=39749) 2024-11-09T00:22:20,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/11fd507eb3414298b6ba7f81fcc5975d 2024-11-09T00:22:20,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cb988f41cfa541cdb6e025f2d1da5285 is 50, key is test_row_0/B:col10/1731111738692/Put/seqid=0 2024-11-09T00:22:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741998_1174 (size=12151) 2024-11-09T00:22:20,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cb988f41cfa541cdb6e025f2d1da5285 2024-11-09T00:22:20,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/f77acb88e10e4780bdea4924c9bd72c8 is 50, key is test_row_0/C:col10/1731111738692/Put/seqid=0 2024-11-09T00:22:20,492 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/2fcd5c722ba444a5a22ac13d1fcf2fe5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/2fcd5c722ba444a5a22ac13d1fcf2fe5 2024-11-09T00:22:20,501 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/C of c510ab9dfa980c00b700ca744b2587a4 into 2fcd5c722ba444a5a22ac13d1fcf2fe5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:20,501 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:20,501 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/C, priority=12, startTime=1731111739854; duration=0sec 2024-11-09T00:22:20,501 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:20,501 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:20,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741999_1175 (size=12151) 2024-11-09T00:22:20,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111800594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:20,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:20,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111800855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,857 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111800865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111800865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111800865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:20,867 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,867 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,868 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:20,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/f77acb88e10e4780bdea4924c9bd72c8 2024-11-09T00:22:20,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/11fd507eb3414298b6ba7f81fcc5975d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d 2024-11-09T00:22:20,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d, entries=200, sequenceid=198, filesize=38.8 K 2024-11-09T00:22:20,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cb988f41cfa541cdb6e025f2d1da5285 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285 2024-11-09T00:22:20,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285, entries=150, sequenceid=198, filesize=11.9 K 2024-11-09T00:22:20,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/f77acb88e10e4780bdea4924c9bd72c8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8 2024-11-09T00:22:20,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8, entries=150, sequenceid=198, filesize=11.9 K 2024-11-09T00:22:20,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c510ab9dfa980c00b700ca744b2587a4 in 1073ms, sequenceid=198, compaction requested=false 2024-11-09T00:22:20,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:21,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T00:22:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:21,013 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:21,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110974c9d4c71bb245bf9de39532d45b2d1c_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111739933/Put/seqid=0 2024-11-09T00:22:21,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742000_1176 (size=12304) 2024-11-09T00:22:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:21,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:21,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111801187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111801295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:21,448 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110974c9d4c71bb245bf9de39532d45b2d1c_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110974c9d4c71bb245bf9de39532d45b2d1c_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:21,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/53aef6ab822242ceb504f5ee7bbdddbe, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/53aef6ab822242ceb504f5ee7bbdddbe is 175, key is test_row_0/A:col10/1731111739933/Put/seqid=0 2024-11-09T00:22:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742001_1177 (size=31105) 2024-11-09T00:22:21,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111801504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111801815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:21,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/53aef6ab822242ceb504f5ee7bbdddbe 2024-11-09T00:22:21,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/b2766ff9001440b9b46f113f1a5d4bfa is 50, key is test_row_0/B:col10/1731111739933/Put/seqid=0 2024-11-09T00:22:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742002_1178 (size=12151) 2024-11-09T00:22:21,870 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/b2766ff9001440b9b46f113f1a5d4bfa 2024-11-09T00:22:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/3327fc5c08e142f09cbae2614099d007 is 50, key is test_row_0/C:col10/1731111739933/Put/seqid=0 2024-11-09T00:22:21,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742003_1179 (size=12151) 2024-11-09T00:22:22,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/3327fc5c08e142f09cbae2614099d007 2024-11-09T00:22:22,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111802335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/53aef6ab822242ceb504f5ee7bbdddbe as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe 2024-11-09T00:22:22,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe, entries=150, sequenceid=214, filesize=30.4 K 2024-11-09T00:22:22,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/b2766ff9001440b9b46f113f1a5d4bfa as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa 2024-11-09T00:22:22,421 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T00:22:22,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/3327fc5c08e142f09cbae2614099d007 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007 2024-11-09T00:22:22,428 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T00:22:22,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c510ab9dfa980c00b700ca744b2587a4 in 1416ms, sequenceid=214, compaction requested=true 2024-11-09T00:22:22,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:22,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:22,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T00:22:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T00:22:22,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T00:22:22,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1350 sec 2024-11-09T00:22:22,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.1490 sec 2024-11-09T00:22:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:22,875 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:22,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:22:22,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b074014e39a844bea0d41e76bc7ebca9_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:22,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742004_1180 (size=14794) 2024-11-09T00:22:22,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111802895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:22,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111802898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111803004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111803014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111803014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111803016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111803214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111803225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111803225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111803226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:23,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b074014e39a844bea0d41e76bc7ebca9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b074014e39a844bea0d41e76bc7ebca9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:23,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/824ca67d2d264958b761483a3f93609a, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/824ca67d2d264958b761483a3f93609a is 175, key is test_row_0/A:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:23,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742005_1181 (size=39749) 2024-11-09T00:22:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111803355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,364 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T00:22:23,465 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:23,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T00:22:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T00:22:23,474 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:23,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:23,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111803524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111803535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111803535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111803535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T00:22:23,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T00:22:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/824ca67d2d264958b761483a3f93609a 2024-11-09T00:22:23,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/3e9f06fd50c649b0866e31b30f4cbd61 is 50, key is test_row_0/B:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:23,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742006_1182 (size=12151) 2024-11-09T00:22:23,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/3e9f06fd50c649b0866e31b30f4cbd61 2024-11-09T00:22:23,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/085f14a9826d4ddd82ce47a14985dd19 is 50, key is test_row_0/C:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:23,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742007_1183 (size=12151) 2024-11-09T00:22:23,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/085f14a9826d4ddd82ce47a14985dd19 2024-11-09T00:22:23,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/824ca67d2d264958b761483a3f93609a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a 2024-11-09T00:22:23,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a, entries=200, sequenceid=238, filesize=38.8 K 2024-11-09T00:22:23,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/3e9f06fd50c649b0866e31b30f4cbd61 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61 2024-11-09T00:22:23,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T00:22:23,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/085f14a9826d4ddd82ce47a14985dd19 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19 2024-11-09T00:22:23,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T00:22:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:23,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T00:22:23,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c510ab9dfa980c00b700ca744b2587a4 in 910ms, sequenceid=238, compaction requested=true 2024-11-09T00:22:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:23,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:23,786 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:23,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:23,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:23,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:23,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:23,790 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:23,790 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:23,790 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:23,790 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:23,790 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,790 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,790 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3396c938709543249055a26fbd5c3705, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.8 K 2024-11-09T00:22:23,790 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=138.8 K 2024-11-09T00:22:23,791 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,791 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a] 2024-11-09T00:22:23,791 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 3396c938709543249055a26fbd5c3705, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737510 2024-11-09T00:22:23,791 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77ed50bd2d940c5953b58fd6cd1553c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737510 2024-11-09T00:22:23,791 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cb988f41cfa541cdb6e025f2d1da5285, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731111738692 2024-11-09T00:22:23,791 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fd507eb3414298b6ba7f81fcc5975d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731111738690 2024-11-09T00:22:23,792 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b2766ff9001440b9b46f113f1a5d4bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731111739933 2024-11-09T00:22:23,792 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53aef6ab822242ceb504f5ee7bbdddbe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731111739933 2024-11-09T00:22:23,792 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9f06fd50c649b0866e31b30f4cbd61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:23,792 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824ca67d2d264958b761483a3f93609a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T00:22:23,807 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:23,807 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/19a68363ceb049dda8331906fa066188 is 50, key is test_row_0/B:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:23,813 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,821 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109aa882e530dff4320ab59c7fe6daae12d_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,825 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109aa882e530dff4320ab59c7fe6daae12d_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,825 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aa882e530dff4320ab59c7fe6daae12d_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742009_1185 (size=4469) 2024-11-09T00:22:23,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742008_1184 (size=12663) 2024-11-09T00:22:23,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T00:22:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:23,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:22:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109853736652f9a404c8f160ffbfea49ab9_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111742895/Put/seqid=0 2024-11-09T00:22:23,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742010_1186 (size=12304) 2024-11-09T00:22:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:23,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109853736652f9a404c8f160ffbfea49ab9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109853736652f9a404c8f160ffbfea49ab9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/6ab6a87272c44d0e909a61ed1ec32e3f, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/6ab6a87272c44d0e909a61ed1ec32e3f is 175, key is test_row_0/A:col10/1731111742895/Put/seqid=0 2024-11-09T00:22:23,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742011_1187 (size=31105) 2024-11-09T00:22:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:24,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111804090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T00:22:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111804098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111804098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111804100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111804205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111804214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111804214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111804214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,241 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:24,241 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/8ab9f9748bb64ed3a13931b6aa51a869 is 175, key is test_row_0/A:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742012_1188 (size=31617) 2024-11-09T00:22:24,254 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/19a68363ceb049dda8331906fa066188 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/19a68363ceb049dda8331906fa066188 2024-11-09T00:22:24,263 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 19a68363ceb049dda8331906fa066188(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:24,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:24,263 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=12, startTime=1731111743786; duration=0sec 2024-11-09T00:22:24,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:24,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:24,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:24,268 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:24,268 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/C is initiating minor compaction (all files) 2024-11-09T00:22:24,268 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/C in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,268 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/2fcd5c722ba444a5a22ac13d1fcf2fe5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=47.8 K 2024-11-09T00:22:24,268 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcd5c722ba444a5a22ac13d1fcf2fe5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731111737510 2024-11-09T00:22:24,269 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f77acb88e10e4780bdea4924c9bd72c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731111738692 2024-11-09T00:22:24,269 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 3327fc5c08e142f09cbae2614099d007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731111739933 2024-11-09T00:22:24,269 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 085f14a9826d4ddd82ce47a14985dd19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:24,274 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/8ab9f9748bb64ed3a13931b6aa51a869 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869 2024-11-09T00:22:24,286 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into 8ab9f9748bb64ed3a13931b6aa51a869(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:24,286 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:24,286 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=12, startTime=1731111743786; duration=0sec 2024-11-09T00:22:24,286 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:24,286 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:24,287 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:24,287 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/080b98a83fef4df3973b147fab75c925 is 50, key is test_row_0/C:col10/1731111741178/Put/seqid=0 2024-11-09T00:22:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742013_1189 (size=12663) 2024-11-09T00:22:24,320 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/080b98a83fef4df3973b147fab75c925 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/080b98a83fef4df3973b147fab75c925 2024-11-09T00:22:24,326 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/C of c510ab9dfa980c00b700ca744b2587a4 into 080b98a83fef4df3973b147fab75c925(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:24,326 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:24,326 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/C, priority=12, startTime=1731111743787; duration=0sec 2024-11-09T00:22:24,326 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:24,326 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:24,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/6ab6a87272c44d0e909a61ed1ec32e3f 2024-11-09T00:22:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cf3d955cbfd441f4849266da4ab08f00 is 50, key is test_row_0/B:col10/1731111742895/Put/seqid=0 2024-11-09T00:22:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742014_1190 (size=12151) 2024-11-09T00:22:24,399 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cf3d955cbfd441f4849266da4ab08f00 2024-11-09T00:22:24,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/39e1207b78de40799aabc3ef594fda20 is 50, key is test_row_0/C:col10/1731111742895/Put/seqid=0 2024-11-09T00:22:24,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111804417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742015_1191 (size=12151) 2024-11-09T00:22:24,431 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/39e1207b78de40799aabc3ef594fda20 2024-11-09T00:22:24,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111804435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111804435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111804436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/6ab6a87272c44d0e909a61ed1ec32e3f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f 2024-11-09T00:22:24,451 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f, entries=150, sequenceid=250, filesize=30.4 K 2024-11-09T00:22:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/cf3d955cbfd441f4849266da4ab08f00 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00 2024-11-09T00:22:24,458 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T00:22:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/39e1207b78de40799aabc3ef594fda20 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20 2024-11-09T00:22:24,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T00:22:24,468 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c510ab9dfa980c00b700ca744b2587a4 in 529ms, sequenceid=250, compaction requested=false 2024-11-09T00:22:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T00:22:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T00:22:24,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T00:22:24,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-09T00:22:24,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0080 sec 2024-11-09T00:22:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T00:22:24,615 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:24,615 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-09T00:22:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:24,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:24,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:24,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:24,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:24,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:22:24,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094caecf66756544b4800a0dd792399181_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:24,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742016_1192 (size=14994) 2024-11-09T00:22:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111804755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,766 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094caecf66756544b4800a0dd792399181_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094caecf66756544b4800a0dd792399181_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111804756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111804758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111804758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/75a26ffa3c43423bb177c53db07fa56c, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/75a26ffa3c43423bb177c53db07fa56c is 175, key is test_row_0/A:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:24,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742017_1193 (size=39949) 2024-11-09T00:22:24,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111804865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111804874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111804880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111804880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:24,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:24,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111805074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111805085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111805094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111805094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/75a26ffa3c43423bb177c53db07fa56c 2024-11-09T00:22:25,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/e01b3f36c67843c1a53e5cfd469b7a42 is 50, key is test_row_0/B:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:25,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742018_1194 (size=12301) 2024-11-09T00:22:25,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/e01b3f36c67843c1a53e5cfd469b7a42 2024-11-09T00:22:25,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/6913e65e87ff443db6c91b90733fdae7 is 50, key is test_row_0/C:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:25,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742019_1195 (size=12301) 2024-11-09T00:22:25,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/6913e65e87ff443db6c91b90733fdae7 2024-11-09T00:22:25,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/75a26ffa3c43423bb177c53db07fa56c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c 2024-11-09T00:22:25,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c, entries=200, sequenceid=278, filesize=39.0 K 2024-11-09T00:22:25,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/e01b3f36c67843c1a53e5cfd469b7a42 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42 2024-11-09T00:22:25,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42, entries=150, sequenceid=278, filesize=12.0 K 2024-11-09T00:22:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:25,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/6913e65e87ff443db6c91b90733fdae7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7 2024-11-09T00:22:25,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7, entries=150, sequenceid=278, filesize=12.0 K 2024-11-09T00:22:25,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for c510ab9dfa980c00b700ca744b2587a4 in 520ms, sequenceid=278, compaction requested=true 2024-11-09T00:22:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:25,257 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:25,258 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:25,259 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,259 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,259 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/19a68363ceb049dda8331906fa066188, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=36.2 K 2024-11-09T00:22:25,259 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=100.3 K 2024-11-09T00:22:25,259 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c] 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a68363ceb049dda8331906fa066188, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:25,259 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab9f9748bb64ed3a13931b6aa51a869, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:25,260 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3d955cbfd441f4849266da4ab08f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731111742893 2024-11-09T00:22:25,260 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab6a87272c44d0e909a61ed1ec32e3f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731111742893 2024-11-09T00:22:25,260 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a26ffa3c43423bb177c53db07fa56c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111744088 2024-11-09T00:22:25,260 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e01b3f36c67843c1a53e5cfd469b7a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111744088 2024-11-09T00:22:25,269 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:25,270 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4cf88645ea544c229d512c2c1a3fba70 is 50, key is test_row_0/B:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:25,271 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:25,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742020_1196 (size=12915) 2024-11-09T00:22:25,287 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411098de67074560b4fa88b48890884348056_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:25,290 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411098de67074560b4fa88b48890884348056_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:25,290 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098de67074560b4fa88b48890884348056_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742021_1197 (size=4469) 2024-11-09T00:22:25,336 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#163 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:25,337 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/33c73aa7ce4a446f85dc55967ffb8243 is 175, key is test_row_0/A:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:25,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742022_1198 (size=31869) 2024-11-09T00:22:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:25,376 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:25,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T00:22:25,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d477b43ebfe47719d2298cf99678589_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111744755/Put/seqid=0 2024-11-09T00:22:25,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742023_1199 (size=14994) 2024-11-09T00:22:25,413 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:25,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d477b43ebfe47719d2298cf99678589_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d477b43ebfe47719d2298cf99678589_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:25,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4c8700c7455843ddb7b75beaffb795d6, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:25,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4c8700c7455843ddb7b75beaffb795d6 is 175, key is test_row_0/A:col10/1731111744755/Put/seqid=0 2024-11-09T00:22:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111805415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111805420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111805422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111805423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111805425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742024_1200 (size=39949) 2024-11-09T00:22:25,436 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4c8700c7455843ddb7b75beaffb795d6 2024-11-09T00:22:25,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/a9b9597112bb4139b137a38b74f5de9a is 50, key is test_row_0/B:col10/1731111744755/Put/seqid=0 2024-11-09T00:22:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742025_1201 (size=12301) 2024-11-09T00:22:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111805534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111805534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111805534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111805535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111805535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,685 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4cf88645ea544c229d512c2c1a3fba70 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4cf88645ea544c229d512c2c1a3fba70 2024-11-09T00:22:25,694 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 4cf88645ea544c229d512c2c1a3fba70(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:25,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,694 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:25,694 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=13, startTime=1731111745257; duration=0sec 2024-11-09T00:22:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,694 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,694 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:25,694 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:25,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,696 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:25,696 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/C is initiating minor compaction (all files) 2024-11-09T00:22:25,696 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/C in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,696 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/080b98a83fef4df3973b147fab75c925, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=36.2 K 2024-11-09T00:22:25,697 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 080b98a83fef4df3973b147fab75c925, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731111741175 2024-11-09T00:22:25,697 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e1207b78de40799aabc3ef594fda20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731111742893 2024-11-09T00:22:25,698 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6913e65e87ff443db6c91b90733fdae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111744088 2024-11-09T00:22:25,709 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:25,709 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/c2941ed82ec7407a88cce6e3e1b9e8ac is 50, key is test_row_0/C:col10/1731111744736/Put/seqid=0 2024-11-09T00:22:25,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742026_1202 (size=12915) 2024-11-09T00:22:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111805745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111805746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111805746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111805748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111805748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:25,760 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/33c73aa7ce4a446f85dc55967ffb8243 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243 2024-11-09T00:22:25,766 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into 33c73aa7ce4a446f85dc55967ffb8243(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:25,766 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:25,766 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=13, startTime=1731111745257; duration=0sec 2024-11-09T00:22:25,767 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:25,767 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:25,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:25,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:25,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/a9b9597112bb4139b137a38b74f5de9a 2024-11-09T00:22:25,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d961ae1f21da4e2b898af7117acba82a is 50, key is test_row_0/C:col10/1731111744755/Put/seqid=0 2024-11-09T00:22:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742027_1203 (size=12301) 2024-11-09T00:22:25,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d961ae1f21da4e2b898af7117acba82a 2024-11-09T00:22:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4c8700c7455843ddb7b75beaffb795d6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6 2024-11-09T00:22:25,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6, entries=200, sequenceid=292, filesize=39.0 K 2024-11-09T00:22:25,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/a9b9597112bb4139b137a38b74f5de9a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a 2024-11-09T00:22:25,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T00:22:25,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d961ae1f21da4e2b898af7117acba82a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d961ae1f21da4e2b898af7117acba82a 2024-11-09T00:22:25,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d961ae1f21da4e2b898af7117acba82a, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T00:22:25,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c510ab9dfa980c00b700ca744b2587a4 in 570ms, sequenceid=292, compaction requested=false 2024-11-09T00:22:25,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:26,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T00:22:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:26,003 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T00:22:26,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110909f2079516004696bbb2cdd30d3f2c72_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111745421/Put/seqid=0 2024-11-09T00:22:26,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742028_1204 (size=12454) 2024-11-09T00:22:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. as already flushing 2024-11-09T00:22:26,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111806064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111806065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111806065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111806066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111806070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,130 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/c2941ed82ec7407a88cce6e3e1b9e8ac as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c2941ed82ec7407a88cce6e3e1b9e8ac 2024-11-09T00:22:26,136 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/C of c510ab9dfa980c00b700ca744b2587a4 into c2941ed82ec7407a88cce6e3e1b9e8ac(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:26,136 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:26,136 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/C, priority=13, startTime=1731111745258; duration=0sec 2024-11-09T00:22:26,136 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:26,136 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:26,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111806188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111806188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111806188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111806188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111806404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111806404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111806404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111806404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:26,434 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110909f2079516004696bbb2cdd30d3f2c72_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110909f2079516004696bbb2cdd30d3f2c72_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c871c8bd2b674eaf8aff50f5f6ba1f33, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c871c8bd2b674eaf8aff50f5f6ba1f33 is 175, key is test_row_0/A:col10/1731111745421/Put/seqid=0 2024-11-09T00:22:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742029_1205 (size=31255) 2024-11-09T00:22:26,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111806584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,652 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-09T00:22:26,652 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:26,652 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:26,652 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,655 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-09T00:22:26,655 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:26,655 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,656 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:26,664 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-09T00:22:26,664 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:26,664 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,664 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:26,665 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-09T00:22:26,665 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:26,665 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,665 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:26,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:26,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111806714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111806714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111806714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111806714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:26,849 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c871c8bd2b674eaf8aff50f5f6ba1f33 2024-11-09T00:22:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4bb0ae816a34486e8e7d52dc1bec2052 is 50, key is test_row_0/B:col10/1731111745421/Put/seqid=0 2024-11-09T00:22:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742030_1206 (size=12301) 2024-11-09T00:22:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59192 deadline: 1731111807225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59200 deadline: 1731111807225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731111807226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59180 deadline: 1731111807226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:27,262 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4bb0ae816a34486e8e7d52dc1bec2052 2024-11-09T00:22:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/bc7bd5ca972846d389bad09520465ed2 is 50, key is test_row_0/C:col10/1731111745421/Put/seqid=0 2024-11-09T00:22:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742031_1207 (size=12301) 2024-11-09T00:22:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59238 deadline: 1731111807605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:27,605 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c510ab9dfa980c00b700ca744b2587a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:22:27,674 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/bc7bd5ca972846d389bad09520465ed2 2024-11-09T00:22:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c871c8bd2b674eaf8aff50f5f6ba1f33 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33 2024-11-09T00:22:27,684 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33, entries=150, sequenceid=316, filesize=30.5 K 2024-11-09T00:22:27,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4bb0ae816a34486e8e7d52dc1bec2052 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052 2024-11-09T00:22:27,689 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052, entries=150, sequenceid=316, filesize=12.0 K 2024-11-09T00:22:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/bc7bd5ca972846d389bad09520465ed2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/bc7bd5ca972846d389bad09520465ed2 2024-11-09T00:22:27,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/bc7bd5ca972846d389bad09520465ed2, entries=150, sequenceid=316, filesize=12.0 K 2024-11-09T00:22:27,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c510ab9dfa980c00b700ca744b2587a4 in 1692ms, sequenceid=316, compaction requested=true 2024-11-09T00:22:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-09T00:22:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-09T00:22:27,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T00:22:27,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0760 sec 2024-11-09T00:22:27,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.0840 sec 2024-11-09T00:22:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:28,236 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:28,236 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-09T00:22:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:22:28,236 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:28,236 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-09T00:22:28,236 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,236 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:28,236 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,236 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,236 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:28,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:28,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110903360194c469440398b3f9dbddc51bcc_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_0/A:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:28,246 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-09T00:22:28,247 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:28,247 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-09T00:22:28,247 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,247 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:28,247 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,247 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:28,247 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:28,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:28,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742032_1208 (size=12454) 2024-11-09T00:22:28,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:28,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110903360194c469440398b3f9dbddc51bcc_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903360194c469440398b3f9dbddc51bcc_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:28,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/b40060a671b64c01a9419d9893419720, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:28,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/b40060a671b64c01a9419d9893419720 is 175, key is test_row_0/A:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:28,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742033_1209 (size=31255) 2024-11-09T00:22:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T00:22:28,784 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:29,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/b40060a671b64c01a9419d9893419720 2024-11-09T00:22:29,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4fe416271f814d64bdb291c1f0ab53b9 is 50, key is test_row_0/B:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:29,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742034_1210 (size=12301) 2024-11-09T00:22:29,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4fe416271f814d64bdb291c1f0ab53b9 2024-11-09T00:22:29,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d9add93ce9a845049f2735cf8fd775e7 is 50, key is test_row_0/C:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742035_1211 (size=12301) 2024-11-09T00:22:29,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d9add93ce9a845049f2735cf8fd775e7 2024-11-09T00:22:29,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/b40060a671b64c01a9419d9893419720 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720 2024-11-09T00:22:29,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720, entries=150, sequenceid=333, filesize=30.5 K 2024-11-09T00:22:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/4fe416271f814d64bdb291c1f0ab53b9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9 2024-11-09T00:22:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9, entries=150, sequenceid=333, filesize=12.0 K 2024-11-09T00:22:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/d9add93ce9a845049f2735cf8fd775e7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d9add93ce9a845049f2735cf8fd775e7 2024-11-09T00:22:29,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d9add93ce9a845049f2735cf8fd775e7, entries=150, sequenceid=333, filesize=12.0 K 2024-11-09T00:22:29,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=13.42 KB/13740 for c510ab9dfa980c00b700ca744b2587a4 in 1277ms, sequenceid=333, compaction requested=true 2024-11-09T00:22:29,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:29,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:29,514 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c510ab9dfa980c00b700ca744b2587a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:29,514 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:29,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/A is initiating minor compaction (all files) 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c510ab9dfa980c00b700ca744b2587a4/B is initiating minor compaction (all files) 2024-11-09T00:22:29,515 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/A in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,515 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c510ab9dfa980c00b700ca744b2587a4/B in TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,515 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4cf88645ea544c229d512c2c1a3fba70, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=48.7 K 2024-11-09T00:22:29,515 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp, totalSize=131.2 K 2024-11-09T00:22:29,515 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720] 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf88645ea544c229d512c2c1a3fba70, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111744088 2024-11-09T00:22:29,515 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c73aa7ce4a446f85dc55967ffb8243, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111744088 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b9597112bb4139b137a38b74f5de9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731111744741 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8700c7455843ddb7b75beaffb795d6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731111744741 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb0ae816a34486e8e7d52dc1bec2052, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731111745415 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c871c8bd2b674eaf8aff50f5f6ba1f33, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731111745415 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe416271f814d64bdb291c1f0ab53b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731111746063 2024-11-09T00:22:29,516 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40060a671b64c01a9419d9893419720, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731111746063 2024-11-09T00:22:29,526 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:29,527 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/9f616f0e61f2434c89de0c300958a315 is 50, key is test_row_0/B:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:29,528 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:29,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742036_1212 (size=13051) 2024-11-09T00:22:29,531 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109a402fdf2947246bf8320a71e54f646eb_c510ab9dfa980c00b700ca744b2587a4 store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:29,548 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109a402fdf2947246bf8320a71e54f646eb_c510ab9dfa980c00b700ca744b2587a4, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:29,549 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a402fdf2947246bf8320a71e54f646eb_c510ab9dfa980c00b700ca744b2587a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:29,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742037_1213 (size=4469) 2024-11-09T00:22:29,554 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c510ab9dfa980c00b700ca744b2587a4#A#compaction#175 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:29,554 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c218be8cc5c74398a146f6875402b11b is 175, key is test_row_0/A:col10/1731111748235/Put/seqid=0 2024-11-09T00:22:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742038_1214 (size=32005) 2024-11-09T00:22:29,616 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-09T00:22:29,616 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:29,616 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:29,616 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:29,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3003 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3024 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1325 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3973 rows 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1334 2024-11-09T00:22:29,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4000 rows 2024-11-09T00:22:29,616 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:22:29,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:22:29,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:29,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:29,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:29,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:22:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:29,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111749620"}]},"ts":"1731111749620"} 2024-11-09T00:22:29,622 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:22:29,622 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:22:29,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:22:29,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, UNASSIGN}] 2024-11-09T00:22:29,625 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, UNASSIGN 2024-11-09T00:22:29,626 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:29,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, UNASSIGN because future has completed 2024-11-09T00:22:29,627 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:22:29,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:29,780 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:29,780 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:22:29,780 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing c510ab9dfa980c00b700ca744b2587a4, disabling compactions & flushes 2024-11-09T00:22:29,780 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:29,938 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/9f616f0e61f2434c89de0c300958a315 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/9f616f0e61f2434c89de0c300958a315 2024-11-09T00:22:29,944 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/B of c510ab9dfa980c00b700ca744b2587a4 into 9f616f0e61f2434c89de0c300958a315(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:29,944 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:29,944 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/B, priority=12, startTime=1731111749514; duration=0sec 2024-11-09T00:22:29,944 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:29,944 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:B 2024-11-09T00:22:29,944 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. because compaction request was cancelled 2024-11-09T00:22:29,945 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:C 2024-11-09T00:22:29,964 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/c218be8cc5c74398a146f6875402b11b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c218be8cc5c74398a146f6875402b11b 2024-11-09T00:22:29,969 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c510ab9dfa980c00b700ca744b2587a4/A of c510ab9dfa980c00b700ca744b2587a4 into c218be8cc5c74398a146f6875402b11b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:29,969 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c510ab9dfa980c00b700ca744b2587a4: 2024-11-09T00:22:29,969 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,969 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., storeName=c510ab9dfa980c00b700ca744b2587a4/A, priority=12, startTime=1731111749513; duration=0sec 2024-11-09T00:22:29,969 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,970 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. after waiting 0 ms 2024-11-09T00:22:29,970 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:29,970 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:29,970 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c510ab9dfa980c00b700ca744b2587a4:A 2024-11-09T00:22:29,970 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing c510ab9dfa980c00b700ca744b2587a4 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-09T00:22:29,975 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109693bdc3e78bd4f5689d937ff8dcb785b_c510ab9dfa980c00b700ca744b2587a4 is 50, key is test_row_1/A:col10/1731111749615/Put/seqid=0 2024-11-09T00:22:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742039_1215 (size=9914) 2024-11-09T00:22:30,118 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:22:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:30,380 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:30,384 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109693bdc3e78bd4f5689d937ff8dcb785b_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109693bdc3e78bd4f5689d937ff8dcb785b_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:30,385 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4d08089a11da4d4d98bd78479e62221c, store: [table=TestAcidGuarantees family=A region=c510ab9dfa980c00b700ca744b2587a4] 2024-11-09T00:22:30,386 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4d08089a11da4d4d98bd78479e62221c is 175, key is test_row_1/A:col10/1731111749615/Put/seqid=0 2024-11-09T00:22:30,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742040_1216 (size=22561) 2024-11-09T00:22:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:30,790 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4d08089a11da4d4d98bd78479e62221c 2024-11-09T00:22:30,800 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/835f5f165cf4403a9d85d3e32e5f21e7 is 50, key is test_row_1/B:col10/1731111749615/Put/seqid=0 2024-11-09T00:22:30,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742041_1217 (size=9857) 2024-11-09T00:22:31,205 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/835f5f165cf4403a9d85d3e32e5f21e7 2024-11-09T00:22:31,213 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/9c760b5fd9c548fba930976f472124c2 is 50, key is test_row_1/C:col10/1731111749615/Put/seqid=0 2024-11-09T00:22:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742042_1218 (size=9857) 2024-11-09T00:22:31,619 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/9c760b5fd9c548fba930976f472124c2 2024-11-09T00:22:31,627 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/A/4d08089a11da4d4d98bd78479e62221c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4d08089a11da4d4d98bd78479e62221c 2024-11-09T00:22:31,633 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4d08089a11da4d4d98bd78479e62221c, entries=100, sequenceid=341, filesize=22.0 K 2024-11-09T00:22:31,635 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/B/835f5f165cf4403a9d85d3e32e5f21e7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/835f5f165cf4403a9d85d3e32e5f21e7 2024-11-09T00:22:31,640 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/835f5f165cf4403a9d85d3e32e5f21e7, entries=100, sequenceid=341, filesize=9.6 K 2024-11-09T00:22:31,641 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/.tmp/C/9c760b5fd9c548fba930976f472124c2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/9c760b5fd9c548fba930976f472124c2 2024-11-09T00:22:31,646 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/9c760b5fd9c548fba930976f472124c2, entries=100, sequenceid=341, filesize=9.6 K 2024-11-09T00:22:31,647 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c510ab9dfa980c00b700ca744b2587a4 in 1677ms, sequenceid=341, compaction requested=true 2024-11-09T00:22:31,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720] to archive 2024-11-09T00:22:31,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:31,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/ed37b5b795b645adbc03ef54f079cf9d 2024-11-09T00:22:31,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/237e38bc97a7469290ab083f1d155064 2024-11-09T00:22:31,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/93f6ace979cb42fd8ca75ad718f110c5 2024-11-09T00:22:31,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/1e00cfc70dc44c43986f47cf0b970782 2024-11-09T00:22:31,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/3a817781fc974c32b6f66d448183b2e3 2024-11-09T00:22:31,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/870517740608460a868301c4a4aa0d8e 2024-11-09T00:22:31,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/9dd11da23c154413b9a057a329845425 2024-11-09T00:22:31,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/803f88d16131479d9fa35729247d45e6 2024-11-09T00:22:31,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/7027e006b11d479fbfebb3fd18f74d6f 2024-11-09T00:22:31,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/333ed85247444dfca80fcb09d5743e88 2024-11-09T00:22:31,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c3f7a58648f646ad8667723dfb20e6c1 2024-11-09T00:22:31,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/e77ed50bd2d940c5953b58fd6cd1553c 2024-11-09T00:22:31,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/11fd507eb3414298b6ba7f81fcc5975d 2024-11-09T00:22:31,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/53aef6ab822242ceb504f5ee7bbdddbe 2024-11-09T00:22:31,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/824ca67d2d264958b761483a3f93609a 2024-11-09T00:22:31,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/8ab9f9748bb64ed3a13931b6aa51a869 2024-11-09T00:22:31,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/6ab6a87272c44d0e909a61ed1ec32e3f 2024-11-09T00:22:31,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/75a26ffa3c43423bb177c53db07fa56c 2024-11-09T00:22:31,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/33c73aa7ce4a446f85dc55967ffb8243 2024-11-09T00:22:31,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4c8700c7455843ddb7b75beaffb795d6 2024-11-09T00:22:31,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c871c8bd2b674eaf8aff50f5f6ba1f33 2024-11-09T00:22:31,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/b40060a671b64c01a9419d9893419720 2024-11-09T00:22:31,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/646a9ed2628742ebaaf513665e36a477, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/922fdef4e11a4477b404097139513aac, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3396c938709543249055a26fbd5c3705, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/19a68363ceb049dda8331906fa066188, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4cf88645ea544c229d512c2c1a3fba70, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9] to archive 2024-11-09T00:22:31,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:31,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/359224811eb14264927fbba51b94ab46 2024-11-09T00:22:31,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/01dc86efd5684e28b6a07c2318a50820 2024-11-09T00:22:31,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/646a9ed2628742ebaaf513665e36a477 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/646a9ed2628742ebaaf513665e36a477 2024-11-09T00:22:31,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/464e8fadcd3c4e9ca8827a692431f59f 2024-11-09T00:22:31,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/7b8b52d3b3bb474892ddee16b9bfc08e 2024-11-09T00:22:31,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/1c3542f38efb4dd1973effdef4e548a1 2024-11-09T00:22:31,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/922fdef4e11a4477b404097139513aac to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/922fdef4e11a4477b404097139513aac 2024-11-09T00:22:31,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/bb6a55a784404648b6bc8f3d65c1101d 2024-11-09T00:22:31,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/416c338a2af1432481468698485f28fb 2024-11-09T00:22:31,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4eb6fb7db5fa4b7189316807eb7b545d 2024-11-09T00:22:31,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3396c938709543249055a26fbd5c3705 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3396c938709543249055a26fbd5c3705 2024-11-09T00:22:31,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/318a52cbffcc4c0db42035cc3e9d57f6 2024-11-09T00:22:31,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cb988f41cfa541cdb6e025f2d1da5285 2024-11-09T00:22:31,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/b2766ff9001440b9b46f113f1a5d4bfa 2024-11-09T00:22:31,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/19a68363ceb049dda8331906fa066188 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/19a68363ceb049dda8331906fa066188 2024-11-09T00:22:31,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/3e9f06fd50c649b0866e31b30f4cbd61 2024-11-09T00:22:31,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/cf3d955cbfd441f4849266da4ab08f00 2024-11-09T00:22:31,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4cf88645ea544c229d512c2c1a3fba70 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4cf88645ea544c229d512c2c1a3fba70 2024-11-09T00:22:31,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/e01b3f36c67843c1a53e5cfd469b7a42 2024-11-09T00:22:31,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/a9b9597112bb4139b137a38b74f5de9a 2024-11-09T00:22:31,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4bb0ae816a34486e8e7d52dc1bec2052 2024-11-09T00:22:31,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/4fe416271f814d64bdb291c1f0ab53b9 2024-11-09T00:22:31,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/42d0a0f9c359447ea3462ee680f2048f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/08cd592a7bbf4a71a39bdc384c498c2a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/2fcd5c722ba444a5a22ac13d1fcf2fe5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/080b98a83fef4df3973b147fab75c925, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7] to archive 2024-11-09T00:22:31,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:31,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b2dd2605ada7431d93b4809eff9c566d 2024-11-09T00:22:31,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/b7e290fdaced41a8ab5d339e4c477cad 2024-11-09T00:22:31,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/42d0a0f9c359447ea3462ee680f2048f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/42d0a0f9c359447ea3462ee680f2048f 2024-11-09T00:22:31,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cd739ea42e8f43d38fc20adb39a0edc2 2024-11-09T00:22:31,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/43113e7d71bb4613aea3fac84beea887 2024-11-09T00:22:31,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/21fb896544124020aa252fea563aeba0 2024-11-09T00:22:31,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/08cd592a7bbf4a71a39bdc384c498c2a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/08cd592a7bbf4a71a39bdc384c498c2a 2024-11-09T00:22:31,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c3c385fb1fbd4d6db1bfd8b9d1448602 2024-11-09T00:22:31,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0db9ebc475104e60a3a44ddcf2717e91 2024-11-09T00:22:31,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/cff2c16a061d4681b282ecfdb8463db0 2024-11-09T00:22:31,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/2fcd5c722ba444a5a22ac13d1fcf2fe5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/2fcd5c722ba444a5a22ac13d1fcf2fe5 2024-11-09T00:22:31,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/0dabbdf578a64d9caf8fe733e9285829 2024-11-09T00:22:31,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/f77acb88e10e4780bdea4924c9bd72c8 2024-11-09T00:22:31,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/3327fc5c08e142f09cbae2614099d007 2024-11-09T00:22:31,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/080b98a83fef4df3973b147fab75c925 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/080b98a83fef4df3973b147fab75c925 2024-11-09T00:22:31,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/085f14a9826d4ddd82ce47a14985dd19 2024-11-09T00:22:31,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/39e1207b78de40799aabc3ef594fda20 2024-11-09T00:22:31,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/6913e65e87ff443db6c91b90733fdae7 2024-11-09T00:22:31,726 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits/344.seqid, newMaxSeqId=344, maxSeqId=4 2024-11-09T00:22:31,727 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. 2024-11-09T00:22:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for c510ab9dfa980c00b700ca744b2587a4: Waiting for close lock at 1731111749780Running coprocessor pre-close hooks at 1731111749780Disabling compacts and flushes for region at 1731111749780Disabling writes for close at 1731111749970 (+190 ms)Obtaining lock to block concurrent updates at 1731111749970Preparing flush snapshotting stores in c510ab9dfa980c00b700ca744b2587a4 at 1731111749970Finished memstore snapshotting TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731111749970Flushing stores of TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4. at 1731111749970Flushing c510ab9dfa980c00b700ca744b2587a4/A: creating writer at 1731111749970Flushing mob file c510ab9dfa980c00b700ca744b2587a4/A: appending metadata at 1731111749975 (+5 ms)Flushing mob file c510ab9dfa980c00b700ca744b2587a4/A: closing flushed file at 1731111749975Flushing c510ab9dfa980c00b700ca744b2587a4/A: appending metadata at 1731111750385 (+410 ms)Flushing c510ab9dfa980c00b700ca744b2587a4/A: closing flushed file at 1731111750385Flushing c510ab9dfa980c00b700ca744b2587a4/B: creating writer at 1731111750796 (+411 ms)Flushing c510ab9dfa980c00b700ca744b2587a4/B: appending metadata at 1731111750799 (+3 ms)Flushing c510ab9dfa980c00b700ca744b2587a4/B: closing flushed file at 1731111750799Flushing c510ab9dfa980c00b700ca744b2587a4/C: creating writer at 1731111751210 (+411 ms)Flushing c510ab9dfa980c00b700ca744b2587a4/C: appending metadata at 1731111751213 (+3 ms)Flushing c510ab9dfa980c00b700ca744b2587a4/C: closing flushed file at 1731111751213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8a17d4: reopening flushed file at 1731111751625 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e47edc5: reopening flushed file at 1731111751634 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f5db0cc: reopening flushed file at 1731111751640 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c510ab9dfa980c00b700ca744b2587a4 in 1677ms, sequenceid=341, compaction requested=true at 1731111751647 (+7 ms)Writing region close event to WAL at 1731111751723 (+76 ms)Running coprocessor post-close hooks at 1731111751727 (+4 ms)Closed at 1731111751727 2024-11-09T00:22:31,729 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c510ab9dfa980c00b700ca744b2587a4, regionState=CLOSED 2024-11-09T00:22:31,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:31,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T00:22:31,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure c510ab9dfa980c00b700ca744b2587a4, server=ef986b38239e,35861,1731111692607 in 2.1050 sec 2024-11-09T00:22:31,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-09T00:22:31,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c510ab9dfa980c00b700ca744b2587a4, UNASSIGN in 2.1090 sec 2024-11-09T00:22:31,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T00:22:31,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1120 sec 2024-11-09T00:22:31,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111751738"}]},"ts":"1731111751738"} 2024-11-09T00:22:31,739 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:22:31,739 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:22:31,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1220 sec 2024-11-09T00:22:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T00:22:31,775 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:31,775 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:22:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,777 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T00:22:31,778 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,781 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,785 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits] 2024-11-09T00:22:31,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4d08089a11da4d4d98bd78479e62221c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/4d08089a11da4d4d98bd78479e62221c 2024-11-09T00:22:31,790 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c218be8cc5c74398a146f6875402b11b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/A/c218be8cc5c74398a146f6875402b11b 2024-11-09T00:22:31,794 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/835f5f165cf4403a9d85d3e32e5f21e7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/835f5f165cf4403a9d85d3e32e5f21e7 2024-11-09T00:22:31,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/9f616f0e61f2434c89de0c300958a315 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/B/9f616f0e61f2434c89de0c300958a315 2024-11-09T00:22:31,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/9c760b5fd9c548fba930976f472124c2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/9c760b5fd9c548fba930976f472124c2 2024-11-09T00:22:31,799 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/bc7bd5ca972846d389bad09520465ed2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/bc7bd5ca972846d389bad09520465ed2 2024-11-09T00:22:31,800 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c2941ed82ec7407a88cce6e3e1b9e8ac to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/c2941ed82ec7407a88cce6e3e1b9e8ac 2024-11-09T00:22:31,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d961ae1f21da4e2b898af7117acba82a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d961ae1f21da4e2b898af7117acba82a 2024-11-09T00:22:31,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d9add93ce9a845049f2735cf8fd775e7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/C/d9add93ce9a845049f2735cf8fd775e7 2024-11-09T00:22:31,804 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits/344.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4/recovered.edits/344.seqid 2024-11-09T00:22:31,805 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,805 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:22:31,805 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:22:31,806 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T00:22:31,810 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903360194c469440398b3f9dbddc51bcc_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110903360194c469440398b3f9dbddc51bcc_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,811 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110909f2079516004696bbb2cdd30d3f2c72_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110909f2079516004696bbb2cdd30d3f2c72_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,812 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d477b43ebfe47719d2298cf99678589_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d477b43ebfe47719d2298cf99678589_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,813 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c1a7540b8654d7789d6edc1b2c8ee2b_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c1a7540b8654d7789d6edc1b2c8ee2b_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,815 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094caecf66756544b4800a0dd792399181_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094caecf66756544b4800a0dd792399181_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,816 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109693bdc3e78bd4f5689d937ff8dcb785b_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109693bdc3e78bd4f5689d937ff8dcb785b_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,818 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110974c9d4c71bb245bf9de39532d45b2d1c_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110974c9d4c71bb245bf9de39532d45b2d1c_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,819 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983940eec83a74be3bf8365c9bf64384f_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110983940eec83a74be3bf8365c9bf64384f_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,820 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109853736652f9a404c8f160ffbfea49ab9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109853736652f9a404c8f160ffbfea49ab9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,821 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110998b98aa150484efd80142d1cee09f4cc_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110998b98aa150484efd80142d1cee09f4cc_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099e8d68451f574463a6d0380bcf589595_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099e8d68451f574463a6d0380bcf589595_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a004b83a432b4d80bba4b10592e23a78_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a004b83a432b4d80bba4b10592e23a78_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,825 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b074014e39a844bea0d41e76bc7ebca9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b074014e39a844bea0d41e76bc7ebca9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,827 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be28a2ddd1f84a8185528b871a41942f_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be28a2ddd1f84a8185528b871a41942f_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c948804b4c6d4f8894e7346f35b82622_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c948804b4c6d4f8894e7346f35b82622_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbf35bf8ef8d4edf8bd60b24265e1033_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dbf35bf8ef8d4edf8bd60b24265e1033_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f65623e0cf9a4b55956d6a91ee4515f1_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f65623e0cf9a4b55956d6a91ee4515f1_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f83861613e96417c8a52e1e5c6c191d9_c510ab9dfa980c00b700ca744b2587a4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f83861613e96417c8a52e1e5c6c191d9_c510ab9dfa980c00b700ca744b2587a4 2024-11-09T00:22:31,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:22:31,836 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,839 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:22:31,842 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:22:31,844 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,844 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:22:31,844 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111751844"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:31,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:22:31,846 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c510ab9dfa980c00b700ca744b2587a4, NAME => 'TestAcidGuarantees,,1731111723462.c510ab9dfa980c00b700ca744b2587a4.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:22:31,847 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:22:31,847 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111751847"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:31,849 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:22:31,850 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-09T00:22:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T00:22:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:22:31,888 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:31,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-488542015_22 at /127.0.0.1:55534 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-60443270_22 at /127.0.0.1:35218 [Waiting for operation #580] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-488542015_22 at /127.0.0.1:55548 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-60443270_22 at /127.0.0.1:38394 [Waiting for operation #217] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=632 (was 640), ProcessCount=11 (was 11), AvailableMemoryMB=3265 (was 3421) 2024-11-09T00:22:31,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=632, ProcessCount=11, AvailableMemoryMB=3264 2024-11-09T00:22:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:22:31,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:22:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:31,906 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:22:31,906 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:31,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-09T00:22:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:31,907 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:22:31,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742043_1219 (size=959) 2024-11-09T00:22:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:32,081 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T00:22:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:32,355 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:22:32,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742044_1220 (size=53) 2024-11-09T00:22:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:32,763 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:22:32,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c231a066d4afdb89d4102d7ef6e77109, disabling compactions & flushes 2024-11-09T00:22:32,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:32,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:32,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. after waiting 0 ms 2024-11-09T00:22:32,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:32,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:32,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c231a066d4afdb89d4102d7ef6e77109: Waiting for close lock at 1731111752764Disabling compacts and flushes for region at 1731111752764Disabling writes for close at 1731111752764Writing region close event to WAL at 1731111752764Closed at 1731111752764 2024-11-09T00:22:32,765 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:22:32,766 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111752765"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111752765"}]},"ts":"1731111752765"} 2024-11-09T00:22:32,767 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:22:32,768 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:22:32,768 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111752768"}]},"ts":"1731111752768"} 2024-11-09T00:22:32,770 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:22:32,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, ASSIGN}] 2024-11-09T00:22:32,771 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, ASSIGN 2024-11-09T00:22:32,772 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:22:32,923 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c231a066d4afdb89d4102d7ef6e77109, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:32,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, ASSIGN because future has completed 2024-11-09T00:22:32,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:33,081 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:33,082 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:22:33,082 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,082 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:22:33,082 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,082 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,083 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,085 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c231a066d4afdb89d4102d7ef6e77109 columnFamilyName A 2024-11-09T00:22:33,085 DEBUG [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:33,085 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(327): Store=c231a066d4afdb89d4102d7ef6e77109/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:33,085 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,086 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c231a066d4afdb89d4102d7ef6e77109 columnFamilyName B 2024-11-09T00:22:33,086 DEBUG [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:33,087 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(327): Store=c231a066d4afdb89d4102d7ef6e77109/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:33,087 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,088 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c231a066d4afdb89d4102d7ef6e77109 columnFamilyName C 2024-11-09T00:22:33,088 DEBUG [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:33,088 INFO [StoreOpener-c231a066d4afdb89d4102d7ef6e77109-1 {}] regionserver.HStore(327): Store=c231a066d4afdb89d4102d7ef6e77109/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:22:33,088 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,089 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,089 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,090 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,090 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,091 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:22:33,091 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,093 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:22:33,094 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened c231a066d4afdb89d4102d7ef6e77109; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72395722, jitterRate=0.07878032326698303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:22:33,094 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:33,094 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for c231a066d4afdb89d4102d7ef6e77109: Running coprocessor pre-open hook at 1731111753082Writing region info on filesystem at 1731111753082Initializing all the Stores at 1731111753083 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111753083Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111753083Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111753083Cleaning up temporary data from old regions at 1731111753090 (+7 ms)Running coprocessor post-open hooks at 1731111753094 (+4 ms)Region opened successfully at 1731111753094 2024-11-09T00:22:33,095 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., pid=59, masterSystemTime=1731111753078 2024-11-09T00:22:33,096 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:33,096 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:33,097 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c231a066d4afdb89d4102d7ef6e77109, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:33,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:33,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T00:22:33,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 in 173 msec 2024-11-09T00:22:33,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T00:22:33,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, ASSIGN in 330 msec 2024-11-09T00:22:33,103 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:22:33,103 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111753103"}]},"ts":"1731111753103"} 2024-11-09T00:22:33,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:22:33,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:22:33,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-11-09T00:22:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-09T00:22:34,055 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:34,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,058 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbe4196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca66ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,065 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:22:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:22:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2191a235 2024-11-09T00:22:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:22:34,067 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:22:34,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d79a68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d75c48b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf63a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,077 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29692e1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,078 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d2d02c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,080 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa500f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,082 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ff06e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,084 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@886eecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,085 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57794a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,088 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e68501, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,089 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51badf2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72dd7c6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,092 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ac3376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,094 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33365016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14741f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,098 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25180bf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cf6aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:22:34,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:22:34,101 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:22:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:22:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:22:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:22:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:22:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:34,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:22:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d734cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:22:34,104 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,104 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,104 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,104 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,104 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,105 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:34,112 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,112 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-09T00:22:34,113 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:34,113 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:22:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,115 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:34,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:34,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:34,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:22:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:22:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:22:34,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:22:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:22:34,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5ad8021b7678473d8cde2dd0aa860449 is 50, key is test_row_0/A:col10/1731111754118/Put/seqid=0 2024-11-09T00:22:34,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111814160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111814162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111814170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111814170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111814171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742045_1221 (size=12001) 2024-11-09T00:22:34,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5ad8021b7678473d8cde2dd0aa860449 2024-11-09T00:22:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:34,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/b536b501b1e3446f8a5d445a802830fd is 50, key is test_row_0/B:col10/1731111754118/Put/seqid=0 2024-11-09T00:22:34,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:34,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:34,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111814275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111814284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111814284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111814285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111814285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742046_1222 (size=12001) 2024-11-09T00:22:34,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:34,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:34,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:34,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111814485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111814494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111814494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111814494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111814494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/b536b501b1e3446f8a5d445a802830fd 2024-11-09T00:22:34,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/81e7d98f38bd4df1bf0f97b63e0c10c7 is 50, key is test_row_0/C:col10/1731111754118/Put/seqid=0 2024-11-09T00:22:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742047_1223 (size=12001) 2024-11-09T00:22:34,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/81e7d98f38bd4df1bf0f97b63e0c10c7 2024-11-09T00:22:34,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5ad8021b7678473d8cde2dd0aa860449 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449 2024-11-09T00:22:34,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:22:34,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/b536b501b1e3446f8a5d445a802830fd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd 2024-11-09T00:22:34,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:34,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:34,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:22:34,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/81e7d98f38bd4df1bf0f97b63e0c10c7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7 2024-11-09T00:22:34,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:22:34,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c231a066d4afdb89d4102d7ef6e77109 in 639ms, sequenceid=13, compaction requested=false 2024-11-09T00:22:34,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:34,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:34,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:22:34,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5b6fc8676b1947af82b2ed496ec490b3 is 50, key is test_row_0/A:col10/1731111754167/Put/seqid=0 2024-11-09T00:22:34,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111814809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111814809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111814810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111814811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111814815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742048_1224 (size=12001) 2024-11-09T00:22:34,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5b6fc8676b1947af82b2ed496ec490b3 2024-11-09T00:22:34,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c75d16e3d9944978a99cd59ebdca5b94 is 50, key is test_row_0/B:col10/1731111754167/Put/seqid=0 2024-11-09T00:22:34,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:34,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:34,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742049_1225 (size=12001) 2024-11-09T00:22:34,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:34,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:34,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c75d16e3d9944978a99cd59ebdca5b94 2024-11-09T00:22:34,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/25948475de9b4d4aa25d4a6a19c2a671 is 50, key is test_row_0/C:col10/1731111754167/Put/seqid=0 2024-11-09T00:22:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742050_1226 (size=12001) 2024-11-09T00:22:34,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111814924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111814925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111814925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111814925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111814925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:35,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/25948475de9b4d4aa25d4a6a19c2a671 2024-11-09T00:22:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5b6fc8676b1947af82b2ed496ec490b3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3 2024-11-09T00:22:35,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T00:22:35,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c75d16e3d9944978a99cd59ebdca5b94 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94 2024-11-09T00:22:35,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T00:22:35,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/25948475de9b4d4aa25d4a6a19c2a671 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671 2024-11-09T00:22:35,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T00:22:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c231a066d4afdb89d4102d7ef6e77109 in 554ms, sequenceid=38, compaction requested=false 2024-11-09T00:22:35,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:35,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:35,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:22:35,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9142154bdaa9412687bc87b14afdd323 is 50, key is test_row_0/A:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742051_1227 (size=12001) 2024-11-09T00:22:35,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9142154bdaa9412687bc87b14afdd323 2024-11-09T00:22:35,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/56323536e38e4eb6b56a31fed28a7cd3 is 50, key is test_row_0/B:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111815481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111815482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111815483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111815483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111815485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742052_1228 (size=12001) 2024-11-09T00:22:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111815595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111815595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111815595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111815595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111815595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111815805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111815805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111815805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111815805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111815805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/56323536e38e4eb6b56a31fed28a7cd3 2024-11-09T00:22:35,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/53d3906ac23c4d48a16b9f47f3eb406d is 50, key is test_row_0/C:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:35,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742053_1229 (size=12001) 2024-11-09T00:22:35,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:35,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:35,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111816114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111816114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111816125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111816125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111816125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:36,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:36,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/53d3906ac23c4d48a16b9f47f3eb406d 2024-11-09T00:22:36,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9142154bdaa9412687bc87b14afdd323 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323 2024-11-09T00:22:36,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T00:22:36,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/56323536e38e4eb6b56a31fed28a7cd3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3 2024-11-09T00:22:36,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T00:22:36,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/53d3906ac23c4d48a16b9f47f3eb406d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d 2024-11-09T00:22:36,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T00:22:36,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c231a066d4afdb89d4102d7ef6e77109 in 903ms, sequenceid=51, compaction requested=true 2024-11-09T00:22:36,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:36,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:36,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:36,350 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:36,351 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:36,354 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:36,354 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:36,354 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:36,354 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:36,354 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,354 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,354 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=35.2 K 2024-11-09T00:22:36,354 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=35.2 K 2024-11-09T00:22:36,355 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad8021b7678473d8cde2dd0aa860449, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111754115 2024-11-09T00:22:36,355 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6fc8676b1947af82b2ed496ec490b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731111754162 2024-11-09T00:22:36,355 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b536b501b1e3446f8a5d445a802830fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111754115 2024-11-09T00:22:36,356 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c75d16e3d9944978a99cd59ebdca5b94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731111754162 2024-11-09T00:22:36,356 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9142154bdaa9412687bc87b14afdd323, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:36,356 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 56323536e38e4eb6b56a31fed28a7cd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:36,373 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:36,374 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/ce03591a0bfd41b99b11dfc8517ce9f4 is 50, key is test_row_0/A:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:36,378 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:36,379 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/64991e610dd04b66847f95eeb2abb5ba is 50, key is test_row_0/B:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:36,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742054_1230 (size=12104) 2024-11-09T00:22:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742055_1231 (size=12104) 2024-11-09T00:22:36,404 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/64991e610dd04b66847f95eeb2abb5ba as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/64991e610dd04b66847f95eeb2abb5ba 2024-11-09T00:22:36,410 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 64991e610dd04b66847f95eeb2abb5ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:36,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:36,410 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=13, startTime=1731111756350; duration=0sec 2024-11-09T00:22:36,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:36,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:36,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:36,412 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:36,412 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:36,412 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,412 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=35.2 K 2024-11-09T00:22:36,413 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e7d98f38bd4df1bf0f97b63e0c10c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111754115 2024-11-09T00:22:36,413 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 25948475de9b4d4aa25d4a6a19c2a671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731111754162 2024-11-09T00:22:36,414 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d3906ac23c4d48a16b9f47f3eb406d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:36,425 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:36,425 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/e6b4d72594ee4802aaaca41d24e54aea is 50, key is test_row_0/C:col10/1731111754813/Put/seqid=0 2024-11-09T00:22:36,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-09T00:22:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:36,433 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:22:36,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/4c51470640824e00af1881b3fb4f61ad is 50, key is test_row_0/A:col10/1731111755483/Put/seqid=0 2024-11-09T00:22:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742056_1232 (size=12104) 2024-11-09T00:22:36,468 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/e6b4d72594ee4802aaaca41d24e54aea as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/e6b4d72594ee4802aaaca41d24e54aea 2024-11-09T00:22:36,476 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into e6b4d72594ee4802aaaca41d24e54aea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:36,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:36,476 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=13, startTime=1731111756351; duration=0sec 2024-11-09T00:22:36,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:36,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:36,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742057_1233 (size=12001) 2024-11-09T00:22:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:36,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:36,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111816636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111816637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111816638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111816638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111816639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111816745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111816745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111816746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111816746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111816754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,789 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/ce03591a0bfd41b99b11dfc8517ce9f4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/ce03591a0bfd41b99b11dfc8517ce9f4 2024-11-09T00:22:36,798 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into ce03591a0bfd41b99b11dfc8517ce9f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:36,799 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:36,799 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=13, startTime=1731111756350; duration=0sec 2024-11-09T00:22:36,799 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:36,799 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:36,883 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/4c51470640824e00af1881b3fb4f61ad 2024-11-09T00:22:36,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1dfada045be44d9d92ebd138a93b5163 is 50, key is test_row_0/B:col10/1731111755483/Put/seqid=0 2024-11-09T00:22:36,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742058_1234 (size=12001) 2024-11-09T00:22:36,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111816955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111816955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111816955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111816955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:36,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111816965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:22:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111817265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111817265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111817265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111817265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111817274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,355 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1dfada045be44d9d92ebd138a93b5163 2024-11-09T00:22:37,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/90738b193fef441f9050980af9227004 is 50, key is test_row_0/C:col10/1731111755483/Put/seqid=0 2024-11-09T00:22:37,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742059_1235 (size=12001) 2024-11-09T00:22:37,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111817774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111817774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111817774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111817774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,785 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/90738b193fef441f9050980af9227004 2024-11-09T00:22:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111817785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:37,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/4c51470640824e00af1881b3fb4f61ad as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad 2024-11-09T00:22:37,795 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T00:22:37,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1dfada045be44d9d92ebd138a93b5163 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163 2024-11-09T00:22:37,801 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T00:22:37,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/90738b193fef441f9050980af9227004 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004 2024-11-09T00:22:37,807 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T00:22:37,809 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c231a066d4afdb89d4102d7ef6e77109 in 1376ms, sequenceid=75, compaction requested=false 2024-11-09T00:22:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-09T00:22:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-09T00:22:37,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T00:22:37,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6950 sec 2024-11-09T00:22:37,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.7010 sec 2024-11-09T00:22:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T00:22:38,285 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:38,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-09T00:22:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:38,287 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:38,288 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:38,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:38,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-09T00:22:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:38,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:38,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/bd105de0cf3c412abe00b2fd339640d8 is 50, key is test_row_0/A:col10/1731111756635/Put/seqid=0 2024-11-09T00:22:38,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742060_1236 (size=12001) 2024-11-09T00:22:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:38,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:38,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/bd105de0cf3c412abe00b2fd339640d8 2024-11-09T00:22:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/bace994cb2244b1288268c20dd6c61f4 is 50, key is test_row_0/B:col10/1731111756635/Put/seqid=0 2024-11-09T00:22:38,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742061_1237 (size=12001) 2024-11-09T00:22:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111818890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:38,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111818891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111818890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111818896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111818897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111819005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111819005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111819005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111819005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111819005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111819215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111819215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111819215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111819215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111819215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,268 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/bace994cb2244b1288268c20dd6c61f4 2024-11-09T00:22:39,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1cdb3d1d704f4b348bb7069c73d898a7 is 50, key is test_row_0/C:col10/1731111756635/Put/seqid=0 2024-11-09T00:22:39,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742062_1238 (size=12001) 2024-11-09T00:22:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111819524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111819524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111819524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111819524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111819525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:39,685 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1cdb3d1d704f4b348bb7069c73d898a7 2024-11-09T00:22:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/bd105de0cf3c412abe00b2fd339640d8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8 2024-11-09T00:22:39,696 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-09T00:22:39,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/bace994cb2244b1288268c20dd6c61f4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4 2024-11-09T00:22:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4, entries=150, sequenceid=90, filesize=11.7 K 2024-11-09T00:22:39,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1cdb3d1d704f4b348bb7069c73d898a7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7 2024-11-09T00:22:39,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7, entries=150, sequenceid=90, filesize=11.7 K 2024-11-09T00:22:39,712 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c231a066d4afdb89d4102d7ef6e77109 in 1271ms, sequenceid=90, compaction requested=true 2024-11-09T00:22:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T00:22:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T00:22:39,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T00:22:39,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-09T00:22:39,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.4310 sec 2024-11-09T00:22:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:40,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:22:40,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/000db60a43f74c5984f34e97a8a21842 is 50, key is test_row_0/A:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:40,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742063_1239 (size=12001) 2024-11-09T00:22:40,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111820046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111820048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111820048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111820050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111820051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111820164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111820164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111820165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111820165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111820165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111820375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111820375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111820375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111820375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111820376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T00:22:40,435 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:40,436 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-09T00:22:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:40,438 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:40,439 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:40,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:40,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/000db60a43f74c5984f34e97a8a21842 2024-11-09T00:22:40,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/86813102abc04b3abdebd57c4d87ee0a is 50, key is test_row_0/B:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742064_1240 (size=12001) 2024-11-09T00:22:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:40,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T00:22:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111820685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111820685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111820685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111820685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111820685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:40,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T00:22:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:40,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/86813102abc04b3abdebd57c4d87ee0a 2024-11-09T00:22:40,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1500dd0127444420bad5486847d7ec77 is 50, key is test_row_0/C:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:40,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742065_1241 (size=12001) 2024-11-09T00:22:40,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1500dd0127444420bad5486847d7ec77 2024-11-09T00:22:40,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/000db60a43f74c5984f34e97a8a21842 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842 2024-11-09T00:22:40,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T00:22:40,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T00:22:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:40,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/86813102abc04b3abdebd57c4d87ee0a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a 2024-11-09T00:22:40,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T00:22:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/1500dd0127444420bad5486847d7ec77 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77 2024-11-09T00:22:40,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T00:22:40,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c231a066d4afdb89d4102d7ef6e77109 in 882ms, sequenceid=116, compaction requested=true 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:40,920 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:40,920 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:40,922 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:40,922 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:40,922 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:40,922 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:40,922 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,922 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,922 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/64991e610dd04b66847f95eeb2abb5ba, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=47.0 K 2024-11-09T00:22:40,922 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/ce03591a0bfd41b99b11dfc8517ce9f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=47.0 K 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 64991e610dd04b66847f95eeb2abb5ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce03591a0bfd41b99b11dfc8517ce9f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c51470640824e00af1881b3fb4f61ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731111755481 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfada045be44d9d92ebd138a93b5163, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731111755481 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd105de0cf3c412abe00b2fd339640d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731111756635 2024-11-09T00:22:40,923 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting bace994cb2244b1288268c20dd6c61f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731111756635 2024-11-09T00:22:40,924 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000db60a43f74c5984f34e97a8a21842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:40,924 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 86813102abc04b3abdebd57c4d87ee0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:40,936 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:40,936 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/87bf398aac1f46edacbf34578355e967 is 50, key is test_row_0/B:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:40,939 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:40,940 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/61d33d0eee29406e8cd1ce885d073812 is 50, key is test_row_0/A:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:40,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742066_1242 (size=12241) 2024-11-09T00:22:40,982 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/87bf398aac1f46edacbf34578355e967 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87bf398aac1f46edacbf34578355e967 2024-11-09T00:22:40,991 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 87bf398aac1f46edacbf34578355e967(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:40,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:40,991 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=12, startTime=1731111760920; duration=0sec 2024-11-09T00:22:40,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:40,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:40,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:40,993 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:40,993 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:40,993 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:40,993 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/e6b4d72594ee4802aaaca41d24e54aea, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=47.0 K 2024-11-09T00:22:40,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742067_1243 (size=12241) 2024-11-09T00:22:40,996 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b4d72594ee4802aaaca41d24e54aea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731111754813 2024-11-09T00:22:40,996 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 90738b193fef441f9050980af9227004, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731111755481 2024-11-09T00:22:40,997 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdb3d1d704f4b348bb7069c73d898a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731111756635 2024-11-09T00:22:40,998 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1500dd0127444420bad5486847d7ec77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:41,002 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/61d33d0eee29406e8cd1ce885d073812 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/61d33d0eee29406e8cd1ce885d073812 2024-11-09T00:22:41,013 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 61d33d0eee29406e8cd1ce885d073812(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:41,013 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:41,013 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=12, startTime=1731111760920; duration=0sec 2024-11-09T00:22:41,013 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:41,014 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:41,017 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:41,018 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/352fc94c244d4dcd94cfd45e59de6f41 is 50, key is test_row_0/C:col10/1731111760037/Put/seqid=0 2024-11-09T00:22:41,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-09T00:22:41,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:41,058 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T00:22:41,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742068_1244 (size=12241) 2024-11-09T00:22:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:41,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/33ba40f4a616454c8924d98aba884ed5 is 50, key is test_row_0/A:col10/1731111760046/Put/seqid=0 2024-11-09T00:22:41,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742069_1245 (size=9657) 2024-11-09T00:22:41,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111821225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111821231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111821232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111821233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111821233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111821334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111821344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111821345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111821345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111821345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,464 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/352fc94c244d4dcd94cfd45e59de6f41 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/352fc94c244d4dcd94cfd45e59de6f41 2024-11-09T00:22:41,469 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into 352fc94c244d4dcd94cfd45e59de6f41(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:41,469 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:41,469 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=12, startTime=1731111760920; duration=0sec 2024-11-09T00:22:41,469 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:41,469 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:41,522 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/33ba40f4a616454c8924d98aba884ed5 2024-11-09T00:22:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/237a68ec948043748635ce7205c6f703 is 50, key is test_row_0/B:col10/1731111760046/Put/seqid=0 2024-11-09T00:22:41,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742070_1246 (size=9657) 2024-11-09T00:22:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111821545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111821555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111821555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111821555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111821555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111821854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111821865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111821865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111821865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111821865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:41,938 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/237a68ec948043748635ce7205c6f703 2024-11-09T00:22:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b86d4a2d87734095b24572634bd7d155 is 50, key is test_row_0/C:col10/1731111760046/Put/seqid=0 2024-11-09T00:22:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742071_1247 (size=9657) 2024-11-09T00:22:42,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T00:22:42,081 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T00:22:42,355 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b86d4a2d87734095b24572634bd7d155 2024-11-09T00:22:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/33ba40f4a616454c8924d98aba884ed5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5 2024-11-09T00:22:42,368 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5, entries=100, sequenceid=128, filesize=9.4 K 2024-11-09T00:22:42,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111822365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/237a68ec948043748635ce7205c6f703 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703 2024-11-09T00:22:42,376 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703, entries=100, sequenceid=128, filesize=9.4 K 2024-11-09T00:22:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111822374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b86d4a2d87734095b24572634bd7d155 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155 2024-11-09T00:22:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111822374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111822376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:42,382 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155, entries=100, sequenceid=128, filesize=9.4 K 2024-11-09T00:22:42,383 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=167.72 KB/171750 for c231a066d4afdb89d4102d7ef6e77109 in 1325ms, sequenceid=128, compaction requested=false 2024-11-09T00:22:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-09T00:22:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-09T00:22:42,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T00:22:42,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-11-09T00:22:42,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9510 sec 2024-11-09T00:22:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T00:22:42,595 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:42,595 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-09T00:22:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T00:22:42,598 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:42,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:42,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T00:22:42,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T00:22:42,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:42,752 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T00:22:42,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/09141f8bcaae4a2bab7381f370262bc0 is 50, key is test_row_0/A:col10/1731111761229/Put/seqid=0 2024-11-09T00:22:42,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742072_1248 (size=12151) 2024-11-09T00:22:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T00:22:43,171 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/09141f8bcaae4a2bab7381f370262bc0 2024-11-09T00:22:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/39a9d29cec9444afad18dec0db7c0524 is 50, key is test_row_0/B:col10/1731111761229/Put/seqid=0 2024-11-09T00:22:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742073_1249 (size=12151) 2024-11-09T00:22:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T00:22:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:43,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111823390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111823394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111823394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,396 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111823394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111823394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,397 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,397 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,398 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111823504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,585 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/39a9d29cec9444afad18dec0db7c0524 2024-11-09T00:22:43,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5ed9a6a985c840789e9162179a18e283 is 50, key is test_row_0/C:col10/1731111761229/Put/seqid=0 2024-11-09T00:22:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742074_1250 (size=12151) 2024-11-09T00:22:43,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5ed9a6a985c840789e9162179a18e283 2024-11-09T00:22:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/09141f8bcaae4a2bab7381f370262bc0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0 2024-11-09T00:22:43,620 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T00:22:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/39a9d29cec9444afad18dec0db7c0524 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524 2024-11-09T00:22:43,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T00:22:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5ed9a6a985c840789e9162179a18e283 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283 2024-11-09T00:22:43,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T00:22:43,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c231a066d4afdb89d4102d7ef6e77109 in 884ms, sequenceid=157, compaction requested=true 2024-11-09T00:22:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-09T00:22:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-09T00:22:43,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T00:22:43,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-11-09T00:22:43,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0460 sec 2024-11-09T00:22:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:43,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:43,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:22:43,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/063f42119767436eb86e80bdfb15c4d3 is 50, key is test_row_0/A:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742075_1251 (size=14541) 2024-11-09T00:22:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T00:22:43,736 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:43,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T00:22:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:43,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:43,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:43,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:43,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111823824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:43,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:43,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111823934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:44,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:44,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/063f42119767436eb86e80bdfb15c4d3 2024-11-09T00:22:44,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/2ff4f4ea86424accb2c36fbb4096bfe8 is 50, key is test_row_0/B:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742076_1252 (size=12151) 2024-11-09T00:22:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111824145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:44,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111824454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:44,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/2ff4f4ea86424accb2c36fbb4096bfe8 2024-11-09T00:22:44,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/38c98579748742559ba75f1894784b20 is 50, key is test_row_0/C:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:44,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742077_1253 (size=12151) 2024-11-09T00:22:44,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:44,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/38c98579748742559ba75f1894784b20 2024-11-09T00:22:44,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111824964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:44,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/063f42119767436eb86e80bdfb15c4d3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3 2024-11-09T00:22:44,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3, entries=200, sequenceid=168, filesize=14.2 K 2024-11-09T00:22:44,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/2ff4f4ea86424accb2c36fbb4096bfe8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8 2024-11-09T00:22:44,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T00:22:44,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/38c98579748742559ba75f1894784b20 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20 2024-11-09T00:22:44,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T00:22:44,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c231a066d4afdb89d4102d7ef6e77109 in 1268ms, sequenceid=168, compaction requested=true 2024-11-09T00:22:44,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:44,990 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:44,990 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:44,991 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:44,992 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:44,992 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,992 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/61d33d0eee29406e8cd1ce885d073812, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=47.5 K 2024-11-09T00:22:44,992 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:44,992 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:44,992 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:44,993 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87bf398aac1f46edacbf34578355e967, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=45.1 K 2024-11-09T00:22:44,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61d33d0eee29406e8cd1ce885d073812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:44,993 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bf398aac1f46edacbf34578355e967, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:44,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ba40f4a616454c8924d98aba884ed5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731111760046 2024-11-09T00:22:44,994 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 237a68ec948043748635ce7205c6f703, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731111760046 2024-11-09T00:22:44,994 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09141f8bcaae4a2bab7381f370262bc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731111761229 2024-11-09T00:22:44,994 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a9d29cec9444afad18dec0db7c0524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731111761229 2024-11-09T00:22:44,994 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff4f4ea86424accb2c36fbb4096bfe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:44,995 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 063f42119767436eb86e80bdfb15c4d3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:45,006 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:45,007 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9aa9b636bb3b4fdfa7c59e202cfedf79 is 50, key is test_row_0/A:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:45,008 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:45,009 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/3208573c03f3412c8a67dc406682dd63 is 50, key is test_row_0/B:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:45,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742078_1254 (size=12527) 2024-11-09T00:22:45,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742079_1255 (size=12527) 2024-11-09T00:22:45,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T00:22:45,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:45,120 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:22:45,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3838b38e1c24aa196cc9b73ddceacde is 50, key is test_row_0/A:col10/1731111763818/Put/seqid=0 2024-11-09T00:22:45,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742080_1256 (size=12151) 2024-11-09T00:22:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:45,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:45,430 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9aa9b636bb3b4fdfa7c59e202cfedf79 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9aa9b636bb3b4fdfa7c59e202cfedf79 2024-11-09T00:22:45,445 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/3208573c03f3412c8a67dc406682dd63 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/3208573c03f3412c8a67dc406682dd63 2024-11-09T00:22:45,459 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 3208573c03f3412c8a67dc406682dd63(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:45,459 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 9aa9b636bb3b4fdfa7c59e202cfedf79(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:45,459 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=12, startTime=1731111764990; duration=0sec 2024-11-09T00:22:45,459 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=12, startTime=1731111764990; duration=0sec 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:45,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:45,459 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111825424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,459 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,461 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:45,461 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:45,461 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:45,461 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/352fc94c244d4dcd94cfd45e59de6f41, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=45.1 K 2024-11-09T00:22:45,462 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352fc94c244d4dcd94cfd45e59de6f41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731111758889 2024-11-09T00:22:45,462 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86d4a2d87734095b24572634bd7d155, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731111760046 2024-11-09T00:22:45,462 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed9a6a985c840789e9162179a18e283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731111761229 2024-11-09T00:22:45,463 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c98579748742559ba75f1894784b20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111825459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111825460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111825460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,471 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:45,472 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/64c24c0bbfd247b0a8772823bfc7e5f4 is 50, key is test_row_0/C:col10/1731111763376/Put/seqid=0 2024-11-09T00:22:45,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742081_1257 (size=12527) 2024-11-09T00:22:45,543 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3838b38e1c24aa196cc9b73ddceacde 2024-11-09T00:22:45,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/5b8cd97ca8d549d6bd69fae698684d97 is 50, key is test_row_0/B:col10/1731111763818/Put/seqid=0 2024-11-09T00:22:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742082_1258 (size=12151) 2024-11-09T00:22:45,558 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/5b8cd97ca8d549d6bd69fae698684d97 2024-11-09T00:22:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/fb7e4bc9b5734d37820736ae6f03243e is 50, key is test_row_0/C:col10/1731111763818/Put/seqid=0 2024-11-09T00:22:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111825574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111825574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111825575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742083_1259 (size=12151) 2024-11-09T00:22:45,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111825784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111825784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111825784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,882 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/64c24c0bbfd247b0a8772823bfc7e5f4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/64c24c0bbfd247b0a8772823bfc7e5f4 2024-11-09T00:22:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:45,887 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into 64c24c0bbfd247b0a8772823bfc7e5f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:45,887 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:45,887 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=12, startTime=1731111764990; duration=0sec 2024-11-09T00:22:45,887 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:45,887 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:45,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/fb7e4bc9b5734d37820736ae6f03243e 2024-11-09T00:22:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3838b38e1c24aa196cc9b73ddceacde as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde 2024-11-09T00:22:45,985 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T00:22:45,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/5b8cd97ca8d549d6bd69fae698684d97 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97 2024-11-09T00:22:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111825984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:45,988 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:45,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T00:22:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/fb7e4bc9b5734d37820736ae6f03243e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e 2024-11-09T00:22:45,996 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T00:22:45,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c231a066d4afdb89d4102d7ef6e77109 in 877ms, sequenceid=193, compaction requested=false 2024-11-09T00:22:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T00:22:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T00:22:46,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T00:22:46,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2590 sec 2024-11-09T00:22:46,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2650 sec 2024-11-09T00:22:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:46,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:22:46,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/6e792e58714442f395267c7c6f2b036a is 50, key is test_row_0/A:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742084_1260 (size=14541) 2024-11-09T00:22:46,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/6e792e58714442f395267c7c6f2b036a 2024-11-09T00:22:46,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 is 50, key is test_row_0/B:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742085_1261 (size=12151) 2024-11-09T00:22:46,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 2024-11-09T00:22:46,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/424b0f01f3504202b822109b44feeeff is 50, key is test_row_0/C:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111826135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111826138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111826138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742086_1262 (size=12151) 2024-11-09T00:22:46,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/424b0f01f3504202b822109b44feeeff 2024-11-09T00:22:46,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/6e792e58714442f395267c7c6f2b036a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a 2024-11-09T00:22:46,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a, entries=200, sequenceid=209, filesize=14.2 K 2024-11-09T00:22:46,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 2024-11-09T00:22:46,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60, entries=150, sequenceid=209, filesize=11.9 K 2024-11-09T00:22:46,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/424b0f01f3504202b822109b44feeeff as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff 2024-11-09T00:22:46,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff, entries=150, sequenceid=209, filesize=11.9 K 2024-11-09T00:22:46,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c231a066d4afdb89d4102d7ef6e77109 in 81ms, sequenceid=209, compaction requested=true 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:46,177 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:46,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:46,177 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:46,178 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:46,179 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:46,179 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:46,179 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9aa9b636bb3b4fdfa7c59e202cfedf79, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=38.3 K 2024-11-09T00:22:46,179 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aa9b636bb3b4fdfa7c59e202cfedf79, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:46,180 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:46,180 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:46,180 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:46,180 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/3208573c03f3412c8a67dc406682dd63, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.0 K 2024-11-09T00:22:46,180 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3838b38e1c24aa196cc9b73ddceacde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731111763807 2024-11-09T00:22:46,180 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 3208573c03f3412c8a67dc406682dd63, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:46,181 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e792e58714442f395267c7c6f2b036a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765414 2024-11-09T00:22:46,181 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8cd97ca8d549d6bd69fae698684d97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731111763807 2024-11-09T00:22:46,182 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ef9b1ce8f00c41a6b8d8f0740dc7fa60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765427 2024-11-09T00:22:46,194 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:46,195 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/56f1cf1140914accb8266f9030ef57ba is 50, key is test_row_0/A:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,207 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:46,207 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/40ddd92bb29a43b5984d0cec29627fff is 50, key is test_row_0/B:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742087_1263 (size=12629) 2024-11-09T00:22:46,222 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/56f1cf1140914accb8266f9030ef57ba as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/56f1cf1140914accb8266f9030ef57ba 2024-11-09T00:22:46,231 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 56f1cf1140914accb8266f9030ef57ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:46,231 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:46,231 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=13, startTime=1731111766177; duration=0sec 2024-11-09T00:22:46,231 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:46,231 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:46,231 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:46,233 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:46,233 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:46,233 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:46,233 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/64c24c0bbfd247b0a8772823bfc7e5f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.0 K 2024-11-09T00:22:46,234 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c24c0bbfd247b0a8772823bfc7e5f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731111763376 2024-11-09T00:22:46,234 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7e4bc9b5734d37820736ae6f03243e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731111763807 2024-11-09T00:22:46,234 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424b0f01f3504202b822109b44feeeff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765427 2024-11-09T00:22:46,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742088_1264 (size=12629) 2024-11-09T00:22:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:46,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:46,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:22:46,250 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:46,250 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/f5366eb378b84d318bc3c3f7fb1688a6 is 50, key is test_row_0/C:col10/1731111766097/Put/seqid=0 2024-11-09T00:22:46,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742089_1265 (size=12629) 2024-11-09T00:22:46,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/22ee12f0142e4e72b78fa6ad915fcf40 is 50, key is test_row_0/A:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111826263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111826264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111826266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,272 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/f5366eb378b84d318bc3c3f7fb1688a6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f5366eb378b84d318bc3c3f7fb1688a6 2024-11-09T00:22:46,283 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into f5366eb378b84d318bc3c3f7fb1688a6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:46,283 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:46,283 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=13, startTime=1731111766177; duration=0sec 2024-11-09T00:22:46,283 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:46,283 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:46,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742090_1266 (size=12151) 2024-11-09T00:22:46,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/22ee12f0142e4e72b78fa6ad915fcf40 2024-11-09T00:22:46,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/fd17caaed1054f829ee331842cf4f28e is 50, key is test_row_0/B:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:46,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742091_1267 (size=12151) 2024-11-09T00:22:46,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/fd17caaed1054f829ee331842cf4f28e 2024-11-09T00:22:46,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b7d59526629e4cb0ba1541915b8e1611 is 50, key is test_row_0/C:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111826374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111826374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111826374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742092_1268 (size=12151) 2024-11-09T00:22:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111826585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111826585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111826585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,645 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/40ddd92bb29a43b5984d0cec29627fff as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/40ddd92bb29a43b5984d0cec29627fff 2024-11-09T00:22:46,652 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 40ddd92bb29a43b5984d0cec29627fff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:46,652 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:46,652 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=13, startTime=1731111766177; duration=0sec 2024-11-09T00:22:46,652 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:46,652 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:46,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b7d59526629e4cb0ba1541915b8e1611 2024-11-09T00:22:46,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/22ee12f0142e4e72b78fa6ad915fcf40 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40 2024-11-09T00:22:46,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40, entries=150, sequenceid=236, filesize=11.9 K 2024-11-09T00:22:46,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/fd17caaed1054f829ee331842cf4f28e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e 2024-11-09T00:22:46,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-09T00:22:46,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b7d59526629e4cb0ba1541915b8e1611 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611 2024-11-09T00:22:46,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611, entries=150, sequenceid=236, filesize=11.9 K 2024-11-09T00:22:46,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c231a066d4afdb89d4102d7ef6e77109 in 552ms, sequenceid=236, compaction requested=false 2024-11-09T00:22:46,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:46,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:46,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:22:46,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3ed55bb40a048dca6d6c9a020c06f43 is 50, key is test_row_1/A:col10/1731111766895/Put/seqid=0 2024-11-09T00:22:46,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742093_1269 (size=12147) 2024-11-09T00:22:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111826921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111826923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111826924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111827034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111827034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111827034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111827244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111827244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111827244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3ed55bb40a048dca6d6c9a020c06f43 2024-11-09T00:22:47,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/84b1d98b271a4b17861bfac8d3713295 is 50, key is test_row_1/B:col10/1731111766895/Put/seqid=0 2024-11-09T00:22:47,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742094_1270 (size=9757) 2024-11-09T00:22:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111827554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111827555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111827555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:47,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/84b1d98b271a4b17861bfac8d3713295 2024-11-09T00:22:47,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/feba1115093b4028acb5ca8bc3f90dc2 is 50, key is test_row_1/C:col10/1731111766895/Put/seqid=0 2024-11-09T00:22:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742095_1271 (size=9757) 2024-11-09T00:22:47,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/feba1115093b4028acb5ca8bc3f90dc2 2024-11-09T00:22:47,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f3ed55bb40a048dca6d6c9a020c06f43 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43 2024-11-09T00:22:47,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T00:22:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/84b1d98b271a4b17861bfac8d3713295 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295 2024-11-09T00:22:47,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295, entries=100, sequenceid=251, filesize=9.5 K 2024-11-09T00:22:47,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/feba1115093b4028acb5ca8bc3f90dc2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2 2024-11-09T00:22:47,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2, entries=100, sequenceid=251, filesize=9.5 K 2024-11-09T00:22:47,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c231a066d4afdb89d4102d7ef6e77109 in 925ms, sequenceid=251, compaction requested=true 2024-11-09T00:22:47,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:47,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:47,821 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:47,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:47,822 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:47,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:47,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:47,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:47,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:47,823 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:47,823 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:47,823 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:47,823 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/40ddd92bb29a43b5984d0cec29627fff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=33.7 K 2024-11-09T00:22:47,823 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:47,824 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ddd92bb29a43b5984d0cec29627fff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765427 2024-11-09T00:22:47,824 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:47,824 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting fd17caaed1054f829ee331842cf4f28e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731111766134 2024-11-09T00:22:47,824 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:47,824 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/56f1cf1140914accb8266f9030ef57ba, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.1 K 2024-11-09T00:22:47,825 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b1d98b271a4b17861bfac8d3713295, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766895 2024-11-09T00:22:47,825 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f1cf1140914accb8266f9030ef57ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765427 2024-11-09T00:22:47,826 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ee12f0142e4e72b78fa6ad915fcf40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731111766134 2024-11-09T00:22:47,827 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ed55bb40a048dca6d6c9a020c06f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766895 2024-11-09T00:22:47,842 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:47,842 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/87819b807a5541b6aa8101fb3a8cc18c is 50, key is test_row_0/B:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:47,845 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:47,845 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/b705d1671f8f425aa448cd7a527ca3d4 is 50, key is test_row_0/A:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:47,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742096_1272 (size=12731) 2024-11-09T00:22:47,865 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/87819b807a5541b6aa8101fb3a8cc18c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87819b807a5541b6aa8101fb3a8cc18c 2024-11-09T00:22:47,871 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 87819b807a5541b6aa8101fb3a8cc18c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:47,871 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:47,871 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=13, startTime=1731111767821; duration=0sec 2024-11-09T00:22:47,871 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:47,871 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:47,872 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:47,873 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:47,873 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:47,873 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:47,873 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f5366eb378b84d318bc3c3f7fb1688a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=33.7 K 2024-11-09T00:22:47,874 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f5366eb378b84d318bc3c3f7fb1688a6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731111765427 2024-11-09T00:22:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742097_1273 (size=12731) 2024-11-09T00:22:47,875 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d59526629e4cb0ba1541915b8e1611, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731111766134 2024-11-09T00:22:47,875 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting feba1115093b4028acb5ca8bc3f90dc2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766895 2024-11-09T00:22:47,883 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/b705d1671f8f425aa448cd7a527ca3d4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/b705d1671f8f425aa448cd7a527ca3d4 2024-11-09T00:22:47,892 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into b705d1671f8f425aa448cd7a527ca3d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:47,892 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:47,892 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=13, startTime=1731111767821; duration=0sec 2024-11-09T00:22:47,892 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:47,892 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T00:22:47,895 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:47,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T00:22:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:47,897 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#232 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:47,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:47,898 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/d3051d6f9339490cb56052af94ea8f25 is 50, key is test_row_0/C:col10/1731111766247/Put/seqid=0 2024-11-09T00:22:47,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:47,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:47,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742098_1274 (size=12731) 2024-11-09T00:22:47,913 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/d3051d6f9339490cb56052af94ea8f25 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/d3051d6f9339490cb56052af94ea8f25 2024-11-09T00:22:47,918 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into d3051d6f9339490cb56052af94ea8f25(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:47,919 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:47,919 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=13, startTime=1731111767822; duration=0sec 2024-11-09T00:22:47,919 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:47,919 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:47,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:22:48,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/a8409b251d7d4ad598b5c08e8b662340 is 50, key is test_row_0/A:col10/1731111766922/Put/seqid=0 2024-11-09T00:22:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:48,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742099_1275 (size=14741) 2024-11-09T00:22:48,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/a8409b251d7d4ad598b5c08e8b662340 2024-11-09T00:22:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111828033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/19a936c78503411ba96e253c9f3b9f05 is 50, key is test_row_0/B:col10/1731111766922/Put/seqid=0 2024-11-09T00:22:48,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742100_1276 (size=12301) 2024-11-09T00:22:48,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/19a936c78503411ba96e253c9f3b9f05 2024-11-09T00:22:48,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111828064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111828065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111828065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/79f97a1c4ca445b4a084af76574adaf8 is 50, key is test_row_0/C:col10/1731111766922/Put/seqid=0 2024-11-09T00:22:48,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742101_1277 (size=12301) 2024-11-09T00:22:48,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/79f97a1c4ca445b4a084af76574adaf8 2024-11-09T00:22:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/a8409b251d7d4ad598b5c08e8b662340 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340 2024-11-09T00:22:48,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340, entries=200, sequenceid=278, filesize=14.4 K 2024-11-09T00:22:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/19a936c78503411ba96e253c9f3b9f05 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05 2024-11-09T00:22:48,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05, entries=150, sequenceid=278, filesize=12.0 K 2024-11-09T00:22:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/79f97a1c4ca445b4a084af76574adaf8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8 2024-11-09T00:22:48,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8, entries=150, sequenceid=278, filesize=12.0 K 2024-11-09T00:22:48,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c231a066d4afdb89d4102d7ef6e77109 in 116ms, sequenceid=278, compaction requested=false 2024-11-09T00:22:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:48,147 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:48,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:48,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/e135a1fa0cf54bb49f9955075deb7d42 is 50, key is test_row_0/A:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742102_1278 (size=12301) 2024-11-09T00:22:48,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/e135a1fa0cf54bb49f9955075deb7d42 2024-11-09T00:22:48,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ac340ff5e62b4413b86c66afbabbab71 is 50, key is test_row_0/B:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742103_1279 (size=12301) 2024-11-09T00:22:48,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ac340ff5e62b4413b86c66afbabbab71 2024-11-09T00:22:48,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5b47cd27ac90453a8df2cebd97ea51fc is 50, key is test_row_0/C:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742104_1280 (size=12301) 2024-11-09T00:22:48,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5b47cd27ac90453a8df2cebd97ea51fc 2024-11-09T00:22:48,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:48,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/e135a1fa0cf54bb49f9955075deb7d42 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42 2024-11-09T00:22:48,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T00:22:48,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/ac340ff5e62b4413b86c66afbabbab71 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71 2024-11-09T00:22:48,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T00:22:48,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5b47cd27ac90453a8df2cebd97ea51fc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc 2024-11-09T00:22:48,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T00:22:48,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=114.05 KB/116790 for c231a066d4afdb89d4102d7ef6e77109 in 98ms, sequenceid=291, compaction requested=true 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:48,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:48,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:48,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:48,247 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:48,247 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:48,247 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,247 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87819b807a5541b6aa8101fb3a8cc18c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.5 K 2024-11-09T00:22:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:48,249 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 87819b807a5541b6aa8101fb3a8cc18c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766247 2024-11-09T00:22:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:48,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T00:22:48,249 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:48,249 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:48,249 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a936c78503411ba96e253c9f3b9f05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111766921 2024-11-09T00:22:48,249 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,249 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/b705d1671f8f425aa448cd7a527ca3d4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=38.8 K 2024-11-09T00:22:48,249 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ac340ff5e62b4413b86c66afbabbab71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:48,250 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b705d1671f8f425aa448cd7a527ca3d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766247 2024-11-09T00:22:48,251 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8409b251d7d4ad598b5c08e8b662340, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111766920 2024-11-09T00:22:48,252 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e135a1fa0cf54bb49f9955075deb7d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:48,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/122a01c8e1ba4299840aeac71e9aab7b is 50, key is test_row_0/A:col10/1731111768234/Put/seqid=0 2024-11-09T00:22:48,271 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:48,272 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/98e435ac4fa04776b54d753e4e84730d is 50, key is test_row_0/B:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,280 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:48,280 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/6fad8d0591404a2b974c02826d2a0d5c is 50, key is test_row_0/A:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742105_1281 (size=12301) 2024-11-09T00:22:48,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742106_1282 (size=12983) 2024-11-09T00:22:48,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/122a01c8e1ba4299840aeac71e9aab7b 2024-11-09T00:22:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742107_1283 (size=12983) 2024-11-09T00:22:48,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c16142c521d24389a76d0148e961598e is 50, key is test_row_0/B:col10/1731111768234/Put/seqid=0 2024-11-09T00:22:48,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742108_1284 (size=12301) 2024-11-09T00:22:48,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111828340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111828454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111828665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,707 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/98e435ac4fa04776b54d753e4e84730d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/98e435ac4fa04776b54d753e4e84730d 2024-11-09T00:22:48,711 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/6fad8d0591404a2b974c02826d2a0d5c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6fad8d0591404a2b974c02826d2a0d5c 2024-11-09T00:22:48,712 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 98e435ac4fa04776b54d753e4e84730d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:48,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:48,712 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=13, startTime=1731111768245; duration=0sec 2024-11-09T00:22:48,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:48,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:48,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:48,713 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:48,713 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:48,713 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,713 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/d3051d6f9339490cb56052af94ea8f25, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.5 K 2024-11-09T00:22:48,714 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d3051d6f9339490cb56052af94ea8f25, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731111766247 2024-11-09T00:22:48,714 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f97a1c4ca445b4a084af76574adaf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731111766921 2024-11-09T00:22:48,715 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b47cd27ac90453a8df2cebd97ea51fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:48,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c16142c521d24389a76d0148e961598e 2024-11-09T00:22:48,717 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 6fad8d0591404a2b974c02826d2a0d5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:48,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:48,717 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=13, startTime=1731111768245; duration=0sec 2024-11-09T00:22:48,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:48,717 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:48,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/f75585cc5f58436db50fb9488fe3604c is 50, key is test_row_0/C:col10/1731111768234/Put/seqid=0 2024-11-09T00:22:48,727 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#243 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:48,727 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/166c2675872e4e7ca636fbffa0593aee is 50, key is test_row_0/C:col10/1731111768146/Put/seqid=0 2024-11-09T00:22:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742109_1285 (size=12301) 2024-11-09T00:22:48,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742110_1286 (size=12983) 2024-11-09T00:22:48,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111828974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:48,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:48,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:48,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111829074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,077 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111829084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111829084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,088 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,089 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:49,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:49,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/f75585cc5f58436db50fb9488fe3604c 2024-11-09T00:22:49,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/122a01c8e1ba4299840aeac71e9aab7b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b 2024-11-09T00:22:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T00:22:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c16142c521d24389a76d0148e961598e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e 2024-11-09T00:22:49,176 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/166c2675872e4e7ca636fbffa0593aee as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/166c2675872e4e7ca636fbffa0593aee 2024-11-09T00:22:49,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T00:22:49,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/f75585cc5f58436db50fb9488fe3604c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c 2024-11-09T00:22:49,182 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into 166c2675872e4e7ca636fbffa0593aee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:49,182 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:49,182 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=13, startTime=1731111768245; duration=0sec 2024-11-09T00:22:49,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T00:22:49,183 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:49,183 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:49,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c231a066d4afdb89d4102d7ef6e77109 in 934ms, sequenceid=312, compaction requested=false 2024-11-09T00:22:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:49,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T00:22:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:49,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T00:22:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5f23ec10da774c838984b9c81e7463a2 is 50, key is test_row_0/A:col10/1731111768339/Put/seqid=0 2024-11-09T00:22:49,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742111_1287 (size=12301) 2024-11-09T00:22:49,307 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5f23ec10da774c838984b9c81e7463a2 2024-11-09T00:22:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/adb6a35d259347588b01bf14191e8d03 is 50, key is test_row_0/B:col10/1731111768339/Put/seqid=0 2024-11-09T00:22:49,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742112_1288 (size=12301) 2024-11-09T00:22:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:49,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111829538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111829546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111829654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111829654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,721 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/adb6a35d259347588b01bf14191e8d03 2024-11-09T00:22:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9066115a0749482b9202373de6bb6360 is 50, key is test_row_0/C:col10/1731111768339/Put/seqid=0 2024-11-09T00:22:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742113_1289 (size=12301) 2024-11-09T00:22:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111829864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111829864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9066115a0749482b9202373de6bb6360 2024-11-09T00:22:50,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/5f23ec10da774c838984b9c81e7463a2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2 2024-11-09T00:22:50,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2, entries=150, sequenceid=330, filesize=12.0 K 2024-11-09T00:22:50,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/adb6a35d259347588b01bf14191e8d03 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03 2024-11-09T00:22:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03, entries=150, sequenceid=330, filesize=12.0 K 2024-11-09T00:22:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9066115a0749482b9202373de6bb6360 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360 2024-11-09T00:22:50,151 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360, entries=150, sequenceid=330, filesize=12.0 K 2024-11-09T00:22:50,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c231a066d4afdb89d4102d7ef6e77109 in 869ms, sequenceid=330, compaction requested=true 2024-11-09T00:22:50,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:50,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:50,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T00:22:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T00:22:50,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T00:22:50,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2550 sec 2024-11-09T00:22:50,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2620 sec 2024-11-09T00:22:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:50,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:50,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T00:22:50,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/606bb5092835459abf2c78e736647642 is 50, key is test_row_0/A:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742114_1290 (size=14741) 2024-11-09T00:22:50,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111830202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111830202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111830314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111830314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111830525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111830525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/606bb5092835459abf2c78e736647642 2024-11-09T00:22:50,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/cc39b36a5f814150b71ff06834f0c4c7 is 50, key is test_row_0/B:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742115_1291 (size=12301) 2024-11-09T00:22:50,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111830834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111830835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/cc39b36a5f814150b71ff06834f0c4c7 2024-11-09T00:22:51,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 is 50, key is test_row_0/C:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:51,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57110 deadline: 1731111831088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:51,092 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742116_1292 (size=12301) 2024-11-09T00:22:51,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57212 deadline: 1731111831096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:51,100 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57234 deadline: 1731111831112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:51,117 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c231a066d4afdb89d4102d7ef6e77109 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:22:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111831344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111831344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:51,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 2024-11-09T00:22:51,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/606bb5092835459abf2c78e736647642 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642 2024-11-09T00:22:51,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642, entries=200, sequenceid=353, filesize=14.4 K 2024-11-09T00:22:51,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/cc39b36a5f814150b71ff06834f0c4c7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7 2024-11-09T00:22:51,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T00:22:51,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 2024-11-09T00:22:51,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T00:22:51,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c231a066d4afdb89d4102d7ef6e77109 in 1346ms, sequenceid=353, compaction requested=true 2024-11-09T00:22:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:51,524 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:51,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T00:22:51,525 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:51,526 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:51,526 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:51,526 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6fad8d0591404a2b974c02826d2a0d5c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=51.1 K 2024-11-09T00:22:51,526 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/98e435ac4fa04776b54d753e4e84730d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=48.7 K 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e435ac4fa04776b54d753e4e84730d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:51,526 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fad8d0591404a2b974c02826d2a0d5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c16142c521d24389a76d0148e961598e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731111768234 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 122a01c8e1ba4299840aeac71e9aab7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731111768234 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb6a35d259347588b01bf14191e8d03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731111768301 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f23ec10da774c838984b9c81e7463a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731111768301 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc39b36a5f814150b71ff06834f0c4c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:51,527 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 606bb5092835459abf2c78e736647642, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:51,548 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:51,549 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/52b2cf539c0e45d99d4638bebab17cd2 is 50, key is test_row_0/A:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:51,551 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:51,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1fe5433631bd49239972d2eeff37265e is 50, key is test_row_0/B:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:51,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742117_1293 (size=13119) 2024-11-09T00:22:51,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742118_1294 (size=13119) 2024-11-09T00:22:51,982 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/52b2cf539c0e45d99d4638bebab17cd2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/52b2cf539c0e45d99d4638bebab17cd2 2024-11-09T00:22:51,986 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1fe5433631bd49239972d2eeff37265e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1fe5433631bd49239972d2eeff37265e 2024-11-09T00:22:51,990 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 52b2cf539c0e45d99d4638bebab17cd2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:51,990 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:51,990 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=12, startTime=1731111771524; duration=0sec 2024-11-09T00:22:51,990 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:51,990 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:51,990 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:22:51,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:22:51,991 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:51,992 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:51,992 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/166c2675872e4e7ca636fbffa0593aee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=48.7 K 2024-11-09T00:22:51,992 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into 1fe5433631bd49239972d2eeff37265e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:51,992 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:51,992 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=12, startTime=1731111771524; duration=0sec 2024-11-09T00:22:51,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:51,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:51,993 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 166c2675872e4e7ca636fbffa0593aee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731111768016 2024-11-09T00:22:51,995 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f75585cc5f58436db50fb9488fe3604c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731111768234 2024-11-09T00:22:51,995 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9066115a0749482b9202373de6bb6360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731111768301 2024-11-09T00:22:51,995 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b3dfc45cd9da417b8fd7bef2e19bf7a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:52,005 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:52,005 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5347cff97afb4d65a145e65bfeb5448f is 50, key is test_row_0/C:col10/1731111769530/Put/seqid=0 2024-11-09T00:22:52,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742119_1295 (size=13119) 2024-11-09T00:22:52,025 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/5347cff97afb4d65a145e65bfeb5448f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5347cff97afb4d65a145e65bfeb5448f 2024-11-09T00:22:52,032 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into 5347cff97afb4d65a145e65bfeb5448f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:52,032 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:52,032 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=12, startTime=1731111771525; duration=0sec 2024-11-09T00:22:52,032 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:52,032 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T00:22:52,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:52,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T00:22:52,071 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T00:22:52,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:52,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T00:22:52,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T00:22:52,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:52,228 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:22:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9f6728736b2b4cc19ec0a94d7c856c36 is 50, key is test_row_0/A:col10/1731111770182/Put/seqid=0 2024-11-09T00:22:52,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742120_1296 (size=12301) 2024-11-09T00:22:52,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9f6728736b2b4cc19ec0a94d7c856c36 2024-11-09T00:22:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/541caaf3739a427ea669186ef875bb82 is 50, key is test_row_0/B:col10/1731111770182/Put/seqid=0 2024-11-09T00:22:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742121_1297 (size=12301) 2024-11-09T00:22:52,293 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/541caaf3739a427ea669186ef875bb82 2024-11-09T00:22:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/668234f1e85148f6b27e9ebd3c329ad5 is 50, key is test_row_0/C:col10/1731111770182/Put/seqid=0 2024-11-09T00:22:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T00:22:52,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:52,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742122_1298 (size=12301) 2024-11-09T00:22:52,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111832494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111832495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111832604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111832604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T00:22:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111832814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111832815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:52,846 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/668234f1e85148f6b27e9ebd3c329ad5 2024-11-09T00:22:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/9f6728736b2b4cc19ec0a94d7c856c36 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36 2024-11-09T00:22:52,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36, entries=150, sequenceid=369, filesize=12.0 K 2024-11-09T00:22:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/541caaf3739a427ea669186ef875bb82 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82 2024-11-09T00:22:52,863 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82, entries=150, sequenceid=369, filesize=12.0 K 2024-11-09T00:22:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/668234f1e85148f6b27e9ebd3c329ad5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5 2024-11-09T00:22:52,870 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5, entries=150, sequenceid=369, filesize=12.0 K 2024-11-09T00:22:52,873 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c231a066d4afdb89d4102d7ef6e77109 in 645ms, sequenceid=369, compaction requested=false 2024-11-09T00:22:52,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:52,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:52,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T00:22:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T00:22:52,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T00:22:52,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 803 msec 2024-11-09T00:22:52,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 810 msec 2024-11-09T00:22:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:53,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:22:53,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:22:53,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f7c41da4a174430195ff1945027f2969 is 50, key is test_row_0/A:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:53,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742123_1299 (size=14741) 2024-11-09T00:22:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111833165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111833168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T00:22:53,205 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:53,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:22:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T00:22:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:53,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:22:53,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:22:53,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:22:53,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111833274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111833274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:53,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111833484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111833484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:53,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f7c41da4a174430195ff1945027f2969 2024-11-09T00:22:53,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/a757c16d9b154b43b44d2492603ed7a3 is 50, key is test_row_0/B:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:53,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742124_1300 (size=12301) 2024-11-09T00:22:53,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/a757c16d9b154b43b44d2492603ed7a3 2024-11-09T00:22:53,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9d6c27cfc46349fa8ca02fb12742e380 is 50, key is test_row_0/C:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:53,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742125_1301 (size=12301) 2024-11-09T00:22:53,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57126 deadline: 1731111833795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57168 deadline: 1731111833795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:22:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:22:53,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:53,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9d6c27cfc46349fa8ca02fb12742e380 2024-11-09T00:22:53,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/f7c41da4a174430195ff1945027f2969 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969 2024-11-09T00:22:53,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969, entries=200, sequenceid=394, filesize=14.4 K 2024-11-09T00:22:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/a757c16d9b154b43b44d2492603ed7a3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3 2024-11-09T00:22:53,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3, entries=150, sequenceid=394, filesize=12.0 K 2024-11-09T00:22:53,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/9d6c27cfc46349fa8ca02fb12742e380 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380 2024-11-09T00:22:53,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:22:53,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380, entries=150, sequenceid=394, filesize=12.0 K 2024-11-09T00:22:53,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c231a066d4afdb89d4102d7ef6e77109 in 865ms, sequenceid=394, compaction requested=true 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:53,992 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:53,992 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c231a066d4afdb89d4102d7ef6e77109:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:22:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:53,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:53,993 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/A is initiating minor compaction (all files) 2024-11-09T00:22:53,993 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/A in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,993 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/52b2cf539c0e45d99d4638bebab17cd2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=39.2 K 2024-11-09T00:22:53,995 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:53,995 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/B is initiating minor compaction (all files) 2024-11-09T00:22:53,995 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/B in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:53,995 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1fe5433631bd49239972d2eeff37265e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.8 K 2024-11-09T00:22:53,995 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b2cf539c0e45d99d4638bebab17cd2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:53,996 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe5433631bd49239972d2eeff37265e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:53,997 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6728736b2b4cc19ec0a94d7c856c36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731111770182 2024-11-09T00:22:53,997 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 541caaf3739a427ea669186ef875bb82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731111770182 2024-11-09T00:22:53,997 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c41da4a174430195ff1945027f2969, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731111772480 2024-11-09T00:22:53,998 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a757c16d9b154b43b44d2492603ed7a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731111772480 2024-11-09T00:22:54,008 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:54,009 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/29cbecdd570d41a8bfde5a952a025afe is 50, key is test_row_0/A:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:54,012 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:54,013 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c31b9cc644c24b169e632562a203df8e is 50, key is test_row_0/B:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:54,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742126_1302 (size=13221) 2024-11-09T00:22:54,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742127_1303 (size=13221) 2024-11-09T00:22:54,031 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/c31b9cc644c24b169e632562a203df8e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c31b9cc644c24b169e632562a203df8e 2024-11-09T00:22:54,039 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/B of c231a066d4afdb89d4102d7ef6e77109 into c31b9cc644c24b169e632562a203df8e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:54,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:54,039 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/B, priority=13, startTime=1731111773992; duration=0sec 2024-11-09T00:22:54,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:22:54,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:B 2024-11-09T00:22:54,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:22:54,041 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:22:54,041 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c231a066d4afdb89d4102d7ef6e77109/C is initiating minor compaction (all files) 2024-11-09T00:22:54,041 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c231a066d4afdb89d4102d7ef6e77109/C in TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:54,041 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5347cff97afb4d65a145e65bfeb5448f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp, totalSize=36.8 K 2024-11-09T00:22:54,042 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5347cff97afb4d65a145e65bfeb5448f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731111769530 2024-11-09T00:22:54,042 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 668234f1e85148f6b27e9ebd3c329ad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731111770182 2024-11-09T00:22:54,042 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6c27cfc46349fa8ca02fb12742e380, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731111772480 2024-11-09T00:22:54,053 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c231a066d4afdb89d4102d7ef6e77109#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:22:54,054 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/2c3996ee742047b288fe0ec16209bf80 is 50, key is test_row_0/C:col10/1731111772487/Put/seqid=0 2024-11-09T00:22:54,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742128_1304 (size=13221) 2024-11-09T00:22:54,113 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-09T00:22:54,114 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,114 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,114 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,115 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-09T00:22:54,115 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,115 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,115 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,116 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-09T00:22:54,116 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,116 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,116 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-11-09T00:22:54,116 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,116 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,116 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,116 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,117 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-09T00:22:54,117 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,117 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,117 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T00:22:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:54,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:22:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/2047da928b7e4dd58163b4cd81de41ef is 50, key is test_row_0/A:col10/1731111773164/Put/seqid=0 2024-11-09T00:22:54,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742129_1305 (size=12301) 2024-11-09T00:22:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:54,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. as already flushing 2024-11-09T00:22:54,306 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-09T00:22:54,306 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-09T00:22:54,306 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,306 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,306 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:54,306 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,306 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,306 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:54,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:54,419 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/29cbecdd570d41a8bfde5a952a025afe as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/29cbecdd570d41a8bfde5a952a025afe 2024-11-09T00:22:54,424 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/A of c231a066d4afdb89d4102d7ef6e77109 into 29cbecdd570d41a8bfde5a952a025afe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:54,424 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:54,424 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/A, priority=13, startTime=1731111773992; duration=0sec 2024-11-09T00:22:54,424 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:54,424 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:A 2024-11-09T00:22:54,463 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/2c3996ee742047b288fe0ec16209bf80 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/2c3996ee742047b288fe0ec16209bf80 2024-11-09T00:22:54,467 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c231a066d4afdb89d4102d7ef6e77109/C of c231a066d4afdb89d4102d7ef6e77109 into 2c3996ee742047b288fe0ec16209bf80(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:22:54,467 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:54,467 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., storeName=c231a066d4afdb89d4102d7ef6e77109/C, priority=13, startTime=1731111773992; duration=0sec 2024-11-09T00:22:54,467 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:22:54,467 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c231a066d4afdb89d4102d7ef6e77109:C 2024-11-09T00:22:54,549 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/2047da928b7e4dd58163b4cd81de41ef 2024-11-09T00:22:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1d146734018a474dbc6dbd579f717d5f is 50, key is test_row_0/B:col10/1731111773164/Put/seqid=0 2024-11-09T00:22:54,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742130_1306 (size=12301) 2024-11-09T00:22:54,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1d146734018a474dbc6dbd579f717d5f 2024-11-09T00:22:54,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/7364f3acb5464ac3ae98efdb4b48d8b9 is 50, key is test_row_0/C:col10/1731111773164/Put/seqid=0 2024-11-09T00:22:54,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742131_1307 (size=12301) 2024-11-09T00:22:55,128 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-11-09T00:22:55,128 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:55,128 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,129 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:55,138 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-09T00:22:55,139 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:55,139 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,139 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:55,147 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-11-09T00:22:55,147 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:22:55,147 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,147 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:55,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:55,372 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/7364f3acb5464ac3ae98efdb4b48d8b9 2024-11-09T00:22:55,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/2047da928b7e4dd58163b4cd81de41ef as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/2047da928b7e4dd58163b4cd81de41ef 2024-11-09T00:22:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/2047da928b7e4dd58163b4cd81de41ef, entries=150, sequenceid=406, filesize=12.0 K 2024-11-09T00:22:55,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/1d146734018a474dbc6dbd579f717d5f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1d146734018a474dbc6dbd579f717d5f 2024-11-09T00:22:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1d146734018a474dbc6dbd579f717d5f, entries=150, sequenceid=406, filesize=12.0 K 2024-11-09T00:22:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/7364f3acb5464ac3ae98efdb4b48d8b9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/7364f3acb5464ac3ae98efdb4b48d8b9 2024-11-09T00:22:55,392 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/7364f3acb5464ac3ae98efdb4b48d8b9, entries=150, sequenceid=406, filesize=12.0 K 2024-11-09T00:22:55,393 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=33.54 KB/34350 for c231a066d4afdb89d4102d7ef6e77109 in 1251ms, sequenceid=406, compaction requested=false 2024-11-09T00:22:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c231a066d4afdb89d4102d7ef6e77109: 2024-11-09T00:22:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T00:22:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T00:22:55,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T00:22:55,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-11-09T00:22:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1930 sec 2024-11-09T00:22:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T00:22:57,365 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:57,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3604 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3507 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3397 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3540 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3525 2024-11-09T00:22:57,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:22:57,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:22:57,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:22:57,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:57,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:22:57,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:22:57,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:22:57,619 INFO [AsyncFSWAL-0-hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData-prefix:ef986b38239e,38689,1731111691424 {}] wal.AbstractFSWAL(1368): Slow sync cost: 250 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39715,DS-832d4b32-28cf-446d-a540-1a19865fc9da,DISK]] 2024-11-09T00:22:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:57,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111777621"}]},"ts":"1731111777621"} 2024-11-09T00:22:57,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:22:57,623 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:22:57,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:22:57,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, UNASSIGN}] 2024-11-09T00:22:57,626 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, UNASSIGN 2024-11-09T00:22:57,626 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=c231a066d4afdb89d4102d7ef6e77109, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:22:57,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, UNASSIGN because future has completed 2024-11-09T00:22:57,628 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:22:57,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:22:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:57,780 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:57,781 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:22:57,781 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing c231a066d4afdb89d4102d7ef6e77109, disabling compactions & flushes 2024-11-09T00:22:57,781 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:57,781 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:57,781 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. after waiting 0 ms 2024-11-09T00:22:57,781 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:57,781 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing c231a066d4afdb89d4102d7ef6e77109 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T00:22:57,786 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/64a74fa040c1427cb629ec9a3515ab7e is 50, key is test_row_0/A:col10/1731111775136/Put/seqid=0 2024-11-09T00:22:57,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742132_1308 (size=9857) 2024-11-09T00:22:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:58,192 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/64a74fa040c1427cb629ec9a3515ab7e 2024-11-09T00:22:58,199 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/0f8de9c7157240a0a0400a7896afb453 is 50, key is test_row_0/B:col10/1731111775136/Put/seqid=0 2024-11-09T00:22:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742133_1309 (size=9857) 2024-11-09T00:22:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:58,605 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/0f8de9c7157240a0a0400a7896afb453 2024-11-09T00:22:58,613 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/172fe972c24f46f4bc0b597192486eb8 is 50, key is test_row_0/C:col10/1731111775136/Put/seqid=0 2024-11-09T00:22:58,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742134_1310 (size=9857) 2024-11-09T00:22:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:59,018 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/172fe972c24f46f4bc0b597192486eb8 2024-11-09T00:22:59,024 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/A/64a74fa040c1427cb629ec9a3515ab7e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/64a74fa040c1427cb629ec9a3515ab7e 2024-11-09T00:22:59,029 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/64a74fa040c1427cb629ec9a3515ab7e, entries=100, sequenceid=416, filesize=9.6 K 2024-11-09T00:22:59,030 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/B/0f8de9c7157240a0a0400a7896afb453 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/0f8de9c7157240a0a0400a7896afb453 2024-11-09T00:22:59,034 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/0f8de9c7157240a0a0400a7896afb453, entries=100, sequenceid=416, filesize=9.6 K 2024-11-09T00:22:59,035 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/.tmp/C/172fe972c24f46f4bc0b597192486eb8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/172fe972c24f46f4bc0b597192486eb8 2024-11-09T00:22:59,039 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/172fe972c24f46f4bc0b597192486eb8, entries=100, sequenceid=416, filesize=9.6 K 2024-11-09T00:22:59,040 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c231a066d4afdb89d4102d7ef6e77109 in 1259ms, sequenceid=416, compaction requested=true 2024-11-09T00:22:59,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/ce03591a0bfd41b99b11dfc8517ce9f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/61d33d0eee29406e8cd1ce885d073812, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9aa9b636bb3b4fdfa7c59e202cfedf79, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/56f1cf1140914accb8266f9030ef57ba, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/b705d1671f8f425aa448cd7a527ca3d4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6fad8d0591404a2b974c02826d2a0d5c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/52b2cf539c0e45d99d4638bebab17cd2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969] to archive 2024-11-09T00:22:59,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:59,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5ad8021b7678473d8cde2dd0aa860449 2024-11-09T00:22:59,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5b6fc8676b1947af82b2ed496ec490b3 2024-11-09T00:22:59,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/ce03591a0bfd41b99b11dfc8517ce9f4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/ce03591a0bfd41b99b11dfc8517ce9f4 2024-11-09T00:22:59,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9142154bdaa9412687bc87b14afdd323 2024-11-09T00:22:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/4c51470640824e00af1881b3fb4f61ad 2024-11-09T00:22:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/bd105de0cf3c412abe00b2fd339640d8 2024-11-09T00:22:59,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/61d33d0eee29406e8cd1ce885d073812 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/61d33d0eee29406e8cd1ce885d073812 2024-11-09T00:22:59,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/000db60a43f74c5984f34e97a8a21842 2024-11-09T00:22:59,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/33ba40f4a616454c8924d98aba884ed5 2024-11-09T00:22:59,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/09141f8bcaae4a2bab7381f370262bc0 2024-11-09T00:22:59,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/063f42119767436eb86e80bdfb15c4d3 2024-11-09T00:22:59,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9aa9b636bb3b4fdfa7c59e202cfedf79 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9aa9b636bb3b4fdfa7c59e202cfedf79 2024-11-09T00:22:59,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3838b38e1c24aa196cc9b73ddceacde 2024-11-09T00:22:59,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6e792e58714442f395267c7c6f2b036a 2024-11-09T00:22:59,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/56f1cf1140914accb8266f9030ef57ba to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/56f1cf1140914accb8266f9030ef57ba 2024-11-09T00:22:59,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/22ee12f0142e4e72b78fa6ad915fcf40 2024-11-09T00:22:59,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/b705d1671f8f425aa448cd7a527ca3d4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/b705d1671f8f425aa448cd7a527ca3d4 2024-11-09T00:22:59,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f3ed55bb40a048dca6d6c9a020c06f43 2024-11-09T00:22:59,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/a8409b251d7d4ad598b5c08e8b662340 2024-11-09T00:22:59,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6fad8d0591404a2b974c02826d2a0d5c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/6fad8d0591404a2b974c02826d2a0d5c 2024-11-09T00:22:59,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/e135a1fa0cf54bb49f9955075deb7d42 2024-11-09T00:22:59,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/122a01c8e1ba4299840aeac71e9aab7b 2024-11-09T00:22:59,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/5f23ec10da774c838984b9c81e7463a2 2024-11-09T00:22:59,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/606bb5092835459abf2c78e736647642 2024-11-09T00:22:59,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/52b2cf539c0e45d99d4638bebab17cd2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/52b2cf539c0e45d99d4638bebab17cd2 2024-11-09T00:22:59,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/9f6728736b2b4cc19ec0a94d7c856c36 2024-11-09T00:22:59,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/f7c41da4a174430195ff1945027f2969 2024-11-09T00:22:59,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/64991e610dd04b66847f95eeb2abb5ba, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87bf398aac1f46edacbf34578355e967, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/3208573c03f3412c8a67dc406682dd63, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/40ddd92bb29a43b5984d0cec29627fff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87819b807a5541b6aa8101fb3a8cc18c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/98e435ac4fa04776b54d753e4e84730d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1fe5433631bd49239972d2eeff37265e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3] to archive 2024-11-09T00:22:59,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:59,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/b536b501b1e3446f8a5d445a802830fd 2024-11-09T00:22:59,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c75d16e3d9944978a99cd59ebdca5b94 2024-11-09T00:22:59,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/64991e610dd04b66847f95eeb2abb5ba to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/64991e610dd04b66847f95eeb2abb5ba 2024-11-09T00:22:59,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/56323536e38e4eb6b56a31fed28a7cd3 2024-11-09T00:22:59,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1dfada045be44d9d92ebd138a93b5163 2024-11-09T00:22:59,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/bace994cb2244b1288268c20dd6c61f4 2024-11-09T00:22:59,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87bf398aac1f46edacbf34578355e967 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87bf398aac1f46edacbf34578355e967 2024-11-09T00:22:59,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/86813102abc04b3abdebd57c4d87ee0a 2024-11-09T00:22:59,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/237a68ec948043748635ce7205c6f703 2024-11-09T00:22:59,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/39a9d29cec9444afad18dec0db7c0524 2024-11-09T00:22:59,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/3208573c03f3412c8a67dc406682dd63 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/3208573c03f3412c8a67dc406682dd63 2024-11-09T00:22:59,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/2ff4f4ea86424accb2c36fbb4096bfe8 2024-11-09T00:22:59,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/5b8cd97ca8d549d6bd69fae698684d97 2024-11-09T00:22:59,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/40ddd92bb29a43b5984d0cec29627fff to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/40ddd92bb29a43b5984d0cec29627fff 2024-11-09T00:22:59,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ef9b1ce8f00c41a6b8d8f0740dc7fa60 2024-11-09T00:22:59,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/fd17caaed1054f829ee331842cf4f28e 2024-11-09T00:22:59,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87819b807a5541b6aa8101fb3a8cc18c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/87819b807a5541b6aa8101fb3a8cc18c 2024-11-09T00:22:59,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/84b1d98b271a4b17861bfac8d3713295 2024-11-09T00:22:59,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/19a936c78503411ba96e253c9f3b9f05 2024-11-09T00:22:59,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/98e435ac4fa04776b54d753e4e84730d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/98e435ac4fa04776b54d753e4e84730d 2024-11-09T00:22:59,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/ac340ff5e62b4413b86c66afbabbab71 2024-11-09T00:22:59,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c16142c521d24389a76d0148e961598e 2024-11-09T00:22:59,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/adb6a35d259347588b01bf14191e8d03 2024-11-09T00:22:59,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1fe5433631bd49239972d2eeff37265e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1fe5433631bd49239972d2eeff37265e 2024-11-09T00:22:59,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/cc39b36a5f814150b71ff06834f0c4c7 2024-11-09T00:22:59,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/541caaf3739a427ea669186ef875bb82 2024-11-09T00:22:59,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/a757c16d9b154b43b44d2492603ed7a3 2024-11-09T00:22:59,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/e6b4d72594ee4802aaaca41d24e54aea, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/352fc94c244d4dcd94cfd45e59de6f41, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/64c24c0bbfd247b0a8772823bfc7e5f4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f5366eb378b84d318bc3c3f7fb1688a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/d3051d6f9339490cb56052af94ea8f25, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/166c2675872e4e7ca636fbffa0593aee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5347cff97afb4d65a145e65bfeb5448f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380] to archive 2024-11-09T00:22:59,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:22:59,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/81e7d98f38bd4df1bf0f97b63e0c10c7 2024-11-09T00:22:59,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/25948475de9b4d4aa25d4a6a19c2a671 2024-11-09T00:22:59,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/e6b4d72594ee4802aaaca41d24e54aea to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/e6b4d72594ee4802aaaca41d24e54aea 2024-11-09T00:22:59,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/53d3906ac23c4d48a16b9f47f3eb406d 2024-11-09T00:22:59,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/90738b193fef441f9050980af9227004 2024-11-09T00:22:59,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1cdb3d1d704f4b348bb7069c73d898a7 2024-11-09T00:22:59,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/352fc94c244d4dcd94cfd45e59de6f41 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/352fc94c244d4dcd94cfd45e59de6f41 2024-11-09T00:22:59,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/1500dd0127444420bad5486847d7ec77 2024-11-09T00:22:59,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b86d4a2d87734095b24572634bd7d155 2024-11-09T00:22:59,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5ed9a6a985c840789e9162179a18e283 2024-11-09T00:22:59,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/64c24c0bbfd247b0a8772823bfc7e5f4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/64c24c0bbfd247b0a8772823bfc7e5f4 2024-11-09T00:22:59,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/38c98579748742559ba75f1894784b20 2024-11-09T00:22:59,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/fb7e4bc9b5734d37820736ae6f03243e 2024-11-09T00:22:59,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f5366eb378b84d318bc3c3f7fb1688a6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f5366eb378b84d318bc3c3f7fb1688a6 2024-11-09T00:22:59,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/424b0f01f3504202b822109b44feeeff 2024-11-09T00:22:59,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b7d59526629e4cb0ba1541915b8e1611 2024-11-09T00:22:59,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/d3051d6f9339490cb56052af94ea8f25 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/d3051d6f9339490cb56052af94ea8f25 2024-11-09T00:22:59,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/feba1115093b4028acb5ca8bc3f90dc2 2024-11-09T00:22:59,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/79f97a1c4ca445b4a084af76574adaf8 2024-11-09T00:22:59,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/166c2675872e4e7ca636fbffa0593aee to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/166c2675872e4e7ca636fbffa0593aee 2024-11-09T00:22:59,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5b47cd27ac90453a8df2cebd97ea51fc 2024-11-09T00:22:59,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/f75585cc5f58436db50fb9488fe3604c 2024-11-09T00:22:59,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9066115a0749482b9202373de6bb6360 2024-11-09T00:22:59,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5347cff97afb4d65a145e65bfeb5448f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/5347cff97afb4d65a145e65bfeb5448f 2024-11-09T00:22:59,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/b3dfc45cd9da417b8fd7bef2e19bf7a7 2024-11-09T00:22:59,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/668234f1e85148f6b27e9ebd3c329ad5 2024-11-09T00:22:59,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/9d6c27cfc46349fa8ca02fb12742e380 2024-11-09T00:22:59,126 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/recovered.edits/419.seqid, newMaxSeqId=419, maxSeqId=1 2024-11-09T00:22:59,126 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. 2024-11-09T00:22:59,126 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for c231a066d4afdb89d4102d7ef6e77109: Waiting for close lock at 1731111777781Running coprocessor pre-close hooks at 1731111777781Disabling compacts and flushes for region at 1731111777781Disabling writes for close at 1731111777781Obtaining lock to block concurrent updates at 1731111777781Preparing flush snapshotting stores in c231a066d4afdb89d4102d7ef6e77109 at 1731111777781Finished memstore snapshotting TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731111777781Flushing stores of TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109. at 1731111777782 (+1 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/A: creating writer at 1731111777782Flushing c231a066d4afdb89d4102d7ef6e77109/A: appending metadata at 1731111777786 (+4 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/A: closing flushed file at 1731111777786Flushing c231a066d4afdb89d4102d7ef6e77109/B: creating writer at 1731111778196 (+410 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/B: appending metadata at 1731111778198 (+2 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/B: closing flushed file at 1731111778198Flushing c231a066d4afdb89d4102d7ef6e77109/C: creating writer at 1731111778610 (+412 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/C: appending metadata at 1731111778613 (+3 ms)Flushing c231a066d4afdb89d4102d7ef6e77109/C: closing flushed file at 1731111778613Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65e31ad8: reopening flushed file at 1731111779023 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@456f651b: reopening flushed file at 1731111779029 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7923e0a5: reopening flushed file at 1731111779035 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c231a066d4afdb89d4102d7ef6e77109 in 1259ms, sequenceid=416, compaction requested=true at 1731111779040 (+5 ms)Writing region close event to WAL at 1731111779123 (+83 ms)Running coprocessor post-close hooks at 1731111779126 (+3 ms)Closed at 1731111779126 2024-11-09T00:22:59,128 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:59,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=c231a066d4afdb89d4102d7ef6e77109, regionState=CLOSED 2024-11-09T00:22:59,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:22:59,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T00:22:59,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure c231a066d4afdb89d4102d7ef6e77109, server=ef986b38239e,35861,1731111692607 in 1.5030 sec 2024-11-09T00:22:59,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-09T00:22:59,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c231a066d4afdb89d4102d7ef6e77109, UNASSIGN in 1.5090 sec 2024-11-09T00:22:59,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T00:22:59,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5120 sec 2024-11-09T00:22:59,139 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111779139"}]},"ts":"1731111779139"} 2024-11-09T00:22:59,141 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:22:59,141 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:22:59,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7740 sec 2024-11-09T00:22:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T00:22:59,765 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:59,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:22:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,768 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T00:22:59,770 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,773 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:59,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/recovered.edits] 2024-11-09T00:22:59,779 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/2047da928b7e4dd58163b4cd81de41ef to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/2047da928b7e4dd58163b4cd81de41ef 2024-11-09T00:22:59,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/29cbecdd570d41a8bfde5a952a025afe to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/29cbecdd570d41a8bfde5a952a025afe 2024-11-09T00:22:59,782 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/64a74fa040c1427cb629ec9a3515ab7e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/A/64a74fa040c1427cb629ec9a3515ab7e 2024-11-09T00:22:59,784 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/0f8de9c7157240a0a0400a7896afb453 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/0f8de9c7157240a0a0400a7896afb453 2024-11-09T00:22:59,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1d146734018a474dbc6dbd579f717d5f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/1d146734018a474dbc6dbd579f717d5f 2024-11-09T00:22:59,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c31b9cc644c24b169e632562a203df8e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/B/c31b9cc644c24b169e632562a203df8e 2024-11-09T00:22:59,788 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/172fe972c24f46f4bc0b597192486eb8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/172fe972c24f46f4bc0b597192486eb8 2024-11-09T00:22:59,789 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/2c3996ee742047b288fe0ec16209bf80 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/2c3996ee742047b288fe0ec16209bf80 2024-11-09T00:22:59,790 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/7364f3acb5464ac3ae98efdb4b48d8b9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/C/7364f3acb5464ac3ae98efdb4b48d8b9 2024-11-09T00:22:59,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/recovered.edits/419.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109/recovered.edits/419.seqid 2024-11-09T00:22:59,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c231a066d4afdb89d4102d7ef6e77109 2024-11-09T00:22:59,793 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:22:59,795 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,798 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:22:59,800 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:22:59,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:22:59,801 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111779801"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:59,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:22:59,803 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c231a066d4afdb89d4102d7ef6e77109, NAME => 'TestAcidGuarantees,,1731111751904.c231a066d4afdb89d4102d7ef6e77109.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:22:59,803 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:22:59,804 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111779803"}]},"ts":"9223372036854775807"} 2024-11-09T00:22:59,805 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:22:59,806 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-09T00:22:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T00:22:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:22:59,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:22:59,888 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 241), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=682 (was 632) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3108 (was 3264) 2024-11-09T00:22:59,897 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=682, ProcessCount=11, AvailableMemoryMB=3108 2024-11-09T00:22:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:22:59,899 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:22:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:22:59,901 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:22:59,901 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:22:59,901 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-09T00:22:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:22:59,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:22:59,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742135_1311 (size=959) 2024-11-09T00:23:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:23:00,118 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:23:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:23:00,309 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:23:00,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742136_1312 (size=53) 2024-11-09T00:23:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3016adb28f5d0508edaeae670cdf0011, disabling compactions & flushes 2024-11-09T00:23:00,716 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. after waiting 0 ms 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:00,716 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:00,716 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3016adb28f5d0508edaeae670cdf0011: Waiting for close lock at 1731111780716Disabling compacts and flushes for region at 1731111780716Disabling writes for close at 1731111780716Writing region close event to WAL at 1731111780716Closed at 1731111780716 2024-11-09T00:23:00,718 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:23:00,718 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111780718"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111780718"}]},"ts":"1731111780718"} 2024-11-09T00:23:00,721 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:23:00,722 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:23:00,722 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111780722"}]},"ts":"1731111780722"} 2024-11-09T00:23:00,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:23:00,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, ASSIGN}] 2024-11-09T00:23:00,725 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, ASSIGN 2024-11-09T00:23:00,726 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:23:00,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:00,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, ASSIGN because future has completed 2024-11-09T00:23:00,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:01,034 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:01,034 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:23:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:23:01,035 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,035 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:01,035 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,035 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,036 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,038 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName A 2024-11-09T00:23:01,038 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:01,038 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:01,038 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,040 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName B 2024-11-09T00:23:01,040 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:01,040 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:01,040 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,041 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName C 2024-11-09T00:23:01,041 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:01,042 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:01,042 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,042 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,043 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,043 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,043 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,044 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:23:01,045 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,047 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:23:01,047 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 3016adb28f5d0508edaeae670cdf0011; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71179911, jitterRate=0.06066332757472992}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:23:01,047 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:01,048 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 3016adb28f5d0508edaeae670cdf0011: Running coprocessor pre-open hook at 1731111781035Writing region info on filesystem at 1731111781035Initializing all the Stores at 1731111781036 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111781036Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111781036Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111781036Cleaning up temporary data from old regions at 1731111781044 (+8 ms)Running coprocessor post-open hooks at 1731111781047 (+3 ms)Region opened successfully at 1731111781048 (+1 ms) 2024-11-09T00:23:01,049 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., pid=83, masterSystemTime=1731111781031 2024-11-09T00:23:01,051 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:01,051 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:01,051 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:01,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:01,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T00:23:01,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 in 174 msec 2024-11-09T00:23:01,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-09T00:23:01,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, ASSIGN in 331 msec 2024-11-09T00:23:01,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:23:01,057 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111781057"}]},"ts":"1731111781057"} 2024-11-09T00:23:01,058 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:23:01,059 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:23:01,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-09T00:23:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-09T00:23:02,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:02,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:02,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:02,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac1f88c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:02,049 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9d1020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,051 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:02,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:02,054 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:23:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:23:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3143f152 2024-11-09T00:23:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:23:02,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:23:02,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:23:02,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:23:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:02,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742137_1313 (size=995) 2024-11-09T00:23:02,470 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T00:23:02,470 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T00:23:02,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:23:02,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE}] 2024-11-09T00:23:02,478 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE 2024-11-09T00:23:02,479 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:02,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE because future has completed 2024-11-09T00:23:02,481 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:23:02,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:02,634 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,634 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:23:02,634 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 3016adb28f5d0508edaeae670cdf0011, disabling compactions & flushes 2024-11-09T00:23:02,634 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,634 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,634 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. after waiting 0 ms 2024-11-09T00:23:02,634 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,637 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T00:23:02,638 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,638 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 3016adb28f5d0508edaeae670cdf0011: Waiting for close lock at 1731111782634Running coprocessor pre-close hooks at 1731111782634Disabling compacts and flushes for region at 1731111782634Disabling writes for close at 1731111782634Writing region close event to WAL at 1731111782634Running coprocessor post-close hooks at 1731111782638 (+4 ms)Closed at 1731111782638 2024-11-09T00:23:02,638 WARN [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 3016adb28f5d0508edaeae670cdf0011 to self. 2024-11-09T00:23:02,640 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=CLOSED 2024-11-09T00:23:02,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:02,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-09T00:23:02,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 in 162 msec 2024-11-09T00:23:02,647 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE; state=CLOSED, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=true 2024-11-09T00:23:02,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:02,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE because future has completed 2024-11-09T00:23:02,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:02,955 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,955 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:23:02,956 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,956 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:02,956 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,956 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,958 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,959 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName A 2024-11-09T00:23:02,960 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:02,961 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:02,961 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,962 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName B 2024-11-09T00:23:02,962 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:02,963 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:02,963 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,964 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3016adb28f5d0508edaeae670cdf0011 columnFamilyName C 2024-11-09T00:23:02,964 DEBUG [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:02,964 INFO [StoreOpener-3016adb28f5d0508edaeae670cdf0011-1 {}] regionserver.HStore(327): Store=3016adb28f5d0508edaeae670cdf0011/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:02,964 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,965 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,967 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,968 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,968 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,968 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:23:02,970 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,971 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 3016adb28f5d0508edaeae670cdf0011; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66123493, jitterRate=-0.014683172106742859}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:23:02,971 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:02,972 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 3016adb28f5d0508edaeae670cdf0011: Running coprocessor pre-open hook at 1731111782956Writing region info on filesystem at 1731111782956Initializing all the Stores at 1731111782957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111782957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111782957Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111782958 (+1 ms)Cleaning up temporary data from old regions at 1731111782968 (+10 ms)Running coprocessor post-open hooks at 1731111782971 (+3 ms)Region opened successfully at 1731111782972 (+1 ms) 2024-11-09T00:23:02,973 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., pid=88, masterSystemTime=1731111782951 2024-11-09T00:23:02,975 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,975 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:02,976 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=OPEN, openSeqNum=5, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:02,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:02,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-09T00:23:02,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 in 179 msec 2024-11-09T00:23:02,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-09T00:23:02,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, REOPEN/MOVE in 502 msec 2024-11-09T00:23:02,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T00:23:02,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-09T00:23:02,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-09T00:23:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T00:23:02,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:02,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:02,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:02,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:02,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:02,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d782080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:02,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:02,992 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1604415f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:02,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430d9403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:02,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:02,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c29acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:02,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64da483e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:02,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:02,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721d0564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11956c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb11402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bf5096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ade2f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d85004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,017 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bd7d37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,019 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5bf3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21da34a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,022 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab242ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3758de27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:03,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:03,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149665c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:03,036 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be07903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:03,040 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,040 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,041 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,041 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,041 DEBUG [Thread-1444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,041 DEBUG [Thread-1442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,042 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:03,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,043 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,043 DEBUG [Thread-1438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,044 DEBUG [Thread-1440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:03,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-09T00:23:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:03,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,048 DEBUG [Thread-1449 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:03,048 DEBUG [Thread-1449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,052 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:03,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,053 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:03,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:23:03,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:03,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:03,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092293a6612cad48cbaed7b51a1cffb3d9_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111783075/Put/seqid=0 2024-11-09T00:23:03,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742138_1314 (size=19474) 2024-11-09T00:23:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111843175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111843176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111843176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111843179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111843189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111843284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111843296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111843296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111843305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111843309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:03,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111843504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111843514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111843515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111843515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111843516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,532 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:03,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092293a6612cad48cbaed7b51a1cffb3d9_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092293a6612cad48cbaed7b51a1cffb3d9_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:03,538 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8ee40aae6d724dc8b2de12b15e98ecd2, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:03,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8ee40aae6d724dc8b2de12b15e98ecd2 is 175, key is test_row_0/A:col10/1731111783075/Put/seqid=0 2024-11-09T00:23:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742139_1315 (size=56733) 2024-11-09T00:23:03,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8ee40aae6d724dc8b2de12b15e98ecd2 2024-11-09T00:23:03,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7a94d54ab5e74ad19b47f7d29eecf1d4 is 50, key is test_row_0/B:col10/1731111783075/Put/seqid=0 2024-11-09T00:23:03,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742140_1316 (size=12001) 2024-11-09T00:23:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:03,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111843814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111843825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111843825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111843825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111843825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:03,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7a94d54ab5e74ad19b47f7d29eecf1d4 2024-11-09T00:23:04,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8357138838fc41789e38922945720b0c is 50, key is test_row_0/C:col10/1731111783075/Put/seqid=0 2024-11-09T00:23:04,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742141_1317 (size=12001) 2024-11-09T00:23:04,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:04,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111844335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111844335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:04,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111844335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111844345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111844345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:04,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:04,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8357138838fc41789e38922945720b0c 2024-11-09T00:23:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8ee40aae6d724dc8b2de12b15e98ecd2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2 2024-11-09T00:23:04,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2, entries=300, sequenceid=15, filesize=55.4 K 2024-11-09T00:23:04,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7a94d54ab5e74ad19b47f7d29eecf1d4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4 2024-11-09T00:23:04,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T00:23:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8357138838fc41789e38922945720b0c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c 2024-11-09T00:23:04,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T00:23:04,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3016adb28f5d0508edaeae670cdf0011 in 1407ms, sequenceid=15, compaction requested=false 2024-11-09T00:23:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T00:23:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:04,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-09T00:23:04,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:04,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:23:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110954ba77af54f746f18c13d4f772fb8df9_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111783156/Put/seqid=0 2024-11-09T00:23:04,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742142_1318 (size=12154) 2024-11-09T00:23:04,655 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:23:05,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,015 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110954ba77af54f746f18c13d4f772fb8df9_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954ba77af54f746f18c13d4f772fb8df9_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:05,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/923ba80ab5e84f4098096409ac0ad176, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/923ba80ab5e84f4098096409ac0ad176 is 175, key is test_row_0/A:col10/1731111783156/Put/seqid=0 2024-11-09T00:23:05,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742143_1319 (size=30955) 2024-11-09T00:23:05,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/923ba80ab5e84f4098096409ac0ad176 2024-11-09T00:23:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/a8b507e824354fdda70ffc3b56692623 is 50, key is test_row_0/B:col10/1731111783156/Put/seqid=0 2024-11-09T00:23:05,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742144_1320 (size=12001) 2024-11-09T00:23:05,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/a8b507e824354fdda70ffc3b56692623 2024-11-09T00:23:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/7dcfb39106754a618eb5da1ad7c4e68b is 50, key is test_row_0/C:col10/1731111783156/Put/seqid=0 2024-11-09T00:23:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742145_1321 (size=12001) 2024-11-09T00:23:05,072 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/7dcfb39106754a618eb5da1ad7c4e68b 2024-11-09T00:23:05,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/923ba80ab5e84f4098096409ac0ad176 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176 2024-11-09T00:23:05,082 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T00:23:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/a8b507e824354fdda70ffc3b56692623 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623 2024-11-09T00:23:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T00:23:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/7dcfb39106754a618eb5da1ad7c4e68b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b 2024-11-09T00:23:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,104 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T00:23:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,105 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 509ms, sequenceid=40, compaction requested=false 2024-11-09T00:23:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-09T00:23:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-09T00:23:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-09T00:23:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0540 sec 2024-11-09T00:23:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.0670 sec 2024-11-09T00:23:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-09T00:23:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,196 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-09T00:23:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T00:23:05,200 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,201 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T00:23:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-09T00:23:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:05,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-09T00:23:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-09T00:23:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T00:23:05,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-09T00:23:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 163 msec 2024-11-09T00:23:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:05,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c39ad54bfb04c4b98d18ba8458f806b_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742147_1323 (size=24358) 2024-11-09T00:23:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c39ad54bfb04c4b98d18ba8458f806b_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c39ad54bfb04c4b98d18ba8458f806b_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/031c5bea09f3447ba8e692cf5342cf89, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/031c5bea09f3447ba8e692cf5342cf89 is 175, key is test_row_0/A:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742146_1322 (size=73995) 2024-11-09T00:23:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T00:23:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/031c5bea09f3447ba8e692cf5342cf89 2024-11-09T00:23:05,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-09T00:23:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,521 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:05,523 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:05,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6c6e98d6aa994e71810608e59e8e664a is 50, key is test_row_0/B:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742148_1324 (size=23705) 2024-11-09T00:23:05,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6c6e98d6aa994e71810608e59e8e664a 2024-11-09T00:23:05,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/30f7e8dff8d24ddaa5392cb6e5d32dfd is 50, key is test_row_0/C:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742149_1325 (size=12001) 2024-11-09T00:23:05,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/30f7e8dff8d24ddaa5392cb6e5d32dfd 2024-11-09T00:23:05,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/031c5bea09f3447ba8e692cf5342cf89 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89 2024-11-09T00:23:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89, entries=400, sequenceid=52, filesize=72.3 K 2024-11-09T00:23:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6c6e98d6aa994e71810608e59e8e664a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a 2024-11-09T00:23:05,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a, entries=400, sequenceid=52, filesize=23.1 K 2024-11-09T00:23:05,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/30f7e8dff8d24ddaa5392cb6e5d32dfd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd 2024-11-09T00:23:05,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T00:23:05,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=87.22 KB/89310 for 3016adb28f5d0508edaeae670cdf0011 in 118ms, sequenceid=52, compaction requested=true 2024-11-09T00:23:05,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:05,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:05,598 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:05,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:05,599 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:05,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T00:23:05,600 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 161683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:05,600 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:05,600 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,600 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:05,600 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=157.9 K 2024-11-09T00:23:05,600 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:05,600 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,600 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,600 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89] 2024-11-09T00:23:05,600 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=46.6 K 2024-11-09T00:23:05,601 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a94d54ab5e74ad19b47f7d29eecf1d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111783052 2024-11-09T00:23:05,601 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ee40aae6d724dc8b2de12b15e98ecd2, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111783052 2024-11-09T00:23:05,602 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b507e824354fdda70ffc3b56692623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111783156 2024-11-09T00:23:05,602 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923ba80ab5e84f4098096409ac0ad176, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111783156 2024-11-09T00:23:05,602 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6e98d6aa994e71810608e59e8e664a, keycount=400, bloomtype=ROW, size=23.1 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785451 2024-11-09T00:23:05,603 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 031c5bea09f3447ba8e692cf5342cf89, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785451 2024-11-09T00:23:05,622 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#278 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:05,622 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/685f658e07c9473b869cd447db031d55 is 50, key is test_row_0/B:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:05,627 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110932ab6f71706541ffb5456f633b1311a4_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111785573/Put/seqid=0 2024-11-09T00:23:05,645 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411096a7d897b447145b496d02097a0ffa9e3_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,650 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411096a7d897b447145b496d02097a0ffa9e3_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,650 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096a7d897b447145b496d02097a0ffa9e3_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742150_1326 (size=12104) 2024-11-09T00:23:05,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742151_1327 (size=17034) 2024-11-09T00:23:05,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:05,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T00:23:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742152_1328 (size=4469) 2024-11-09T00:23:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,680 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/685f658e07c9473b869cd447db031d55 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/685f658e07c9473b869cd447db031d55 2024-11-09T00:23:05,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110932ab6f71706541ffb5456f633b1311a4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932ab6f71706541ffb5456f633b1311a4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:05,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ab7bbb6ea1f3478d9f8602447c1d8c1d, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:05,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ab7bbb6ea1f3478d9f8602447c1d8c1d is 175, key is test_row_0/A:col10/1731111785573/Put/seqid=0 2024-11-09T00:23:05,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742153_1329 (size=48139) 2024-11-09T00:23:05,689 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 685f658e07c9473b869cd447db031d55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:05,689 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:05,689 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=13, startTime=1731111785599; duration=0sec 2024-11-09T00:23:05,689 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:05,689 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:05,689 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:05,691 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:05,691 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:05,691 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,691 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=35.2 K 2024-11-09T00:23:05,691 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8357138838fc41789e38922945720b0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731111783052 2024-11-09T00:23:05,692 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dcfb39106754a618eb5da1ad7c4e68b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731111783156 2024-11-09T00:23:05,692 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f7e8dff8d24ddaa5392cb6e5d32dfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785476 2024-11-09T00:23:05,701 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:05,701 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/5800b8f0f08b446aaed758d38fc80209 is 50, key is test_row_0/C:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:05,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742154_1330 (size=12104) 2024-11-09T00:23:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111845692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111845693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111845700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,722 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/5800b8f0f08b446aaed758d38fc80209 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5800b8f0f08b446aaed758d38fc80209 2024-11-09T00:23:05,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111845714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111845718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,748 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 5800b8f0f08b446aaed758d38fc80209(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:05,748 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:05,748 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=13, startTime=1731111785599; duration=0sec 2024-11-09T00:23:05,748 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:05,748 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111845815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111845815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T00:23:05,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:05,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111845824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111845845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111845845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:05,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T00:23:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:05,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111846024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111846034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111846045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111846055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111846055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,078 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#279 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:06,079 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/2f490e487b9a4bda8c616e4ee01029ec is 175, key is test_row_0/A:col10/1731111785477/Put/seqid=0 2024-11-09T00:23:06,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=73, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ab7bbb6ea1f3478d9f8602447c1d8c1d 2024-11-09T00:23:06,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742155_1331 (size=31058) 2024-11-09T00:23:06,102 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/2f490e487b9a4bda8c616e4ee01029ec as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec 2024-11-09T00:23:06,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/b2b987123211446e8c7d11050256c254 is 50, key is test_row_0/B:col10/1731111785573/Put/seqid=0 2024-11-09T00:23:06,114 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 2f490e487b9a4bda8c616e4ee01029ec(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:06,114 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:06,114 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=13, startTime=1731111785598; duration=0sec 2024-11-09T00:23:06,114 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:06,114 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:06,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T00:23:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:06,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:06,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742156_1332 (size=12001) 2024-11-09T00:23:06,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/b2b987123211446e8c7d11050256c254 2024-11-09T00:23:06,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/ceb3b25165fb4a2ca6503f324f464ab7 is 50, key is test_row_0/C:col10/1731111785573/Put/seqid=0 2024-11-09T00:23:06,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742157_1333 (size=12001) 2024-11-09T00:23:06,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/ceb3b25165fb4a2ca6503f324f464ab7 2024-11-09T00:23:06,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ab7bbb6ea1f3478d9f8602447c1d8c1d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d 2024-11-09T00:23:06,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d, entries=250, sequenceid=73, filesize=47.0 K 2024-11-09T00:23:06,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/b2b987123211446e8c7d11050256c254 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254 2024-11-09T00:23:06,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254, entries=150, sequenceid=73, filesize=11.7 K 2024-11-09T00:23:06,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/ceb3b25165fb4a2ca6503f324f464ab7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7 2024-11-09T00:23:06,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7, entries=150, sequenceid=73, filesize=11.7 K 2024-11-09T00:23:06,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 3016adb28f5d0508edaeae670cdf0011 in 648ms, sequenceid=73, compaction requested=false 2024-11-09T00:23:06,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:06,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-09T00:23:06,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:06,294 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:23:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d4c27804ab4c49eb8d0fe08dfa525181_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111785691/Put/seqid=0 2024-11-09T00:23:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:06,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:06,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742158_1334 (size=12154) 2024-11-09T00:23:06,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:06,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d4c27804ab4c49eb8d0fe08dfa525181_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d4c27804ab4c49eb8d0fe08dfa525181_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/941808545a5a4858a0b6fb43cfcaf52b, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:06,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/941808545a5a4858a0b6fb43cfcaf52b is 175, key is test_row_0/A:col10/1731111785691/Put/seqid=0 2024-11-09T00:23:06,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742159_1335 (size=30955) 2024-11-09T00:23:06,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111846402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111846404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111846403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111846406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111846406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111846514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111846525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111846525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111846525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111846525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111846724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111846735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111846735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111846735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111846735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:06,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/941808545a5a4858a0b6fb43cfcaf52b 2024-11-09T00:23:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ec57328f519548efa153e52d14332f72 is 50, key is test_row_0/B:col10/1731111785691/Put/seqid=0 2024-11-09T00:23:06,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742160_1336 (size=12001) 2024-11-09T00:23:06,832 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ec57328f519548efa153e52d14332f72 2024-11-09T00:23:06,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8d19738adbd64ce0a78b42dc6239de4d is 50, key is test_row_0/C:col10/1731111785691/Put/seqid=0 2024-11-09T00:23:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742161_1337 (size=12001) 2024-11-09T00:23:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111847045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111847054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111847054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111847054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111847054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8d19738adbd64ce0a78b42dc6239de4d 2024-11-09T00:23:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/941808545a5a4858a0b6fb43cfcaf52b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b 2024-11-09T00:23:07,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b, entries=150, sequenceid=92, filesize=30.2 K 2024-11-09T00:23:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ec57328f519548efa153e52d14332f72 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72 2024-11-09T00:23:07,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T00:23:07,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8d19738adbd64ce0a78b42dc6239de4d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d 2024-11-09T00:23:07,284 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T00:23:07,285 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3016adb28f5d0508edaeae670cdf0011 in 991ms, sequenceid=92, compaction requested=true 2024-11-09T00:23:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-09T00:23:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-09T00:23:07,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-09T00:23:07,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-11-09T00:23:07,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.7750 sec 2024-11-09T00:23:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:07,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:07,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T00:23:07,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094d12e2e03b6345ef90beb0b565bc4770_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:07,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742162_1338 (size=14594) 2024-11-09T00:23:07,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:07,593 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094d12e2e03b6345ef90beb0b565bc4770_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094d12e2e03b6345ef90beb0b565bc4770_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:07,595 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e060d035fcd34b65817e5523374b5c1e, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:07,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e060d035fcd34b65817e5523374b5c1e is 175, key is test_row_0/A:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:07,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742163_1339 (size=39549) 2024-11-09T00:23:07,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e060d035fcd34b65817e5523374b5c1e 2024-11-09T00:23:07,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111847593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/4bcea2af383949b6a6def5c010b4e944 is 50, key is test_row_0/B:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742164_1340 (size=12001) 2024-11-09T00:23:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111847607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111847607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111847608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111847608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T00:23:07,675 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:07,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-09T00:23:07,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:07,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:07,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111847714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111847725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111847735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111847735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111847735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:07,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:07,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111847935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111847944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111847954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111847954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111847954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:07,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:07,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:08,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/4bcea2af383949b6a6def5c010b4e944 2024-11-09T00:23:08,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/5c71ae266145403eb7e041b65b566bf6 is 50, key is test_row_0/C:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:08,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742165_1341 (size=12001) 2024-11-09T00:23:08,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111848245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111848264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111848274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111848275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111848275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:08,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/5c71ae266145403eb7e041b65b566bf6 2024-11-09T00:23:08,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e060d035fcd34b65817e5523374b5c1e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e 2024-11-09T00:23:08,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e, entries=200, sequenceid=113, filesize=38.6 K 2024-11-09T00:23:08,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/4bcea2af383949b6a6def5c010b4e944 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944 2024-11-09T00:23:08,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944, entries=150, sequenceid=113, filesize=11.7 K 2024-11-09T00:23:08,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/5c71ae266145403eb7e041b65b566bf6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6 2024-11-09T00:23:08,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:08,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6, entries=150, sequenceid=113, filesize=11.7 K 2024-11-09T00:23:08,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 3016adb28f5d0508edaeae670cdf0011 in 881ms, sequenceid=113, compaction requested=true 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:08,445 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:08,445 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:08,446 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:08,446 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:08,446 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,446 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=146.2 K 2024-11-09T00:23:08,446 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:08,446 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:08,446 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,446 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,446 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e] 2024-11-09T00:23:08,446 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/685f658e07c9473b869cd447db031d55, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.0 K 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f490e487b9a4bda8c616e4ee01029ec, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785476 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 685f658e07c9473b869cd447db031d55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785476 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7bbb6ea1f3478d9f8602447c1d8c1d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731111785516 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b987123211446e8c7d11050256c254, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731111785573 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941808545a5a4858a0b6fb43cfcaf52b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731111785690 2024-11-09T00:23:08,447 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ec57328f519548efa153e52d14332f72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731111785690 2024-11-09T00:23:08,448 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e060d035fcd34b65817e5523374b5c1e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:08,448 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bcea2af383949b6a6def5c010b4e944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:08,465 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:08,479 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:08,480 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/56b6ef8d108f4b988fb5c856d36789a6 is 50, key is test_row_0/B:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:08,481 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110959245077cf0e4d9484d39629721ec72f_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:08,484 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110959245077cf0e4d9484d39629721ec72f_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:08,484 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110959245077cf0e4d9484d39629721ec72f_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:08,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742166_1342 (size=12241) 2024-11-09T00:23:08,522 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/56b6ef8d108f4b988fb5c856d36789a6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/56b6ef8d108f4b988fb5c856d36789a6 2024-11-09T00:23:08,528 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 56b6ef8d108f4b988fb5c856d36789a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:08,528 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:08,528 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=12, startTime=1731111788445; duration=0sec 2024-11-09T00:23:08,528 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:08,528 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:08,528 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:08,529 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:08,529 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:08,529 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,530 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5800b8f0f08b446aaed758d38fc80209, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.0 K 2024-11-09T00:23:08,530 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5800b8f0f08b446aaed758d38fc80209, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731111785476 2024-11-09T00:23:08,530 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb3b25165fb4a2ca6503f324f464ab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731111785573 2024-11-09T00:23:08,530 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d19738adbd64ce0a78b42dc6239de4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731111785690 2024-11-09T00:23:08,530 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c71ae266145403eb7e041b65b566bf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:08,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742167_1343 (size=4469) 2024-11-09T00:23:08,543 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:08,543 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/74bce3936b55441ebe870753600ca3ec is 50, key is test_row_0/C:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:08,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742168_1344 (size=12241) 2024-11-09T00:23:08,568 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/74bce3936b55441ebe870753600ca3ec as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/74bce3936b55441ebe870753600ca3ec 2024-11-09T00:23:08,577 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 74bce3936b55441ebe870753600ca3ec(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:08,577 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:08,577 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=12, startTime=1731111788445; duration=0sec 2024-11-09T00:23:08,577 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:08,577 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:08,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-09T00:23:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:08,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:23:08,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e46ca644148047f7ae9a59264e2b8325_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111787607/Put/seqid=0 2024-11-09T00:23:08,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742169_1345 (size=12154) 2024-11-09T00:23:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111848812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111848821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111848822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111848825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111848826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,939 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#290 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:08,939 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6671163193a8420bafec3b50686d4f4f is 175, key is test_row_0/A:col10/1731111786402/Put/seqid=0 2024-11-09T00:23:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111848934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111848934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742170_1346 (size=31195) 2024-11-09T00:23:08,949 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6671163193a8420bafec3b50686d4f4f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f 2024-11-09T00:23:08,955 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 6671163193a8420bafec3b50686d4f4f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:08,955 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:08,955 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=12, startTime=1731111788445; duration=0sec 2024-11-09T00:23:08,955 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:08,955 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111848944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111848944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111848945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:09,021 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e46ca644148047f7ae9a59264e2b8325_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e46ca644148047f7ae9a59264e2b8325_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:09,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8acfd557ae414761a76141168d1c9b1b, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:09,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8acfd557ae414761a76141168d1c9b1b is 175, key is test_row_0/A:col10/1731111787607/Put/seqid=0 2024-11-09T00:23:09,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742171_1347 (size=30955) 2024-11-09T00:23:09,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111849155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111849155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111849164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111849164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111849164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,438 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8acfd557ae414761a76141168d1c9b1b 2024-11-09T00:23:09,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/837e6b72eb0d414ca7deead705a43825 is 50, key is test_row_0/B:col10/1731111787607/Put/seqid=0 2024-11-09T00:23:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742172_1348 (size=12001) 2024-11-09T00:23:09,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111849465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111849465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:09,852 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/837e6b72eb0d414ca7deead705a43825 2024-11-09T00:23:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8367c08317b04774ba84b495fd51b535 is 50, key is test_row_0/C:col10/1731111787607/Put/seqid=0 2024-11-09T00:23:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742173_1349 (size=12001) 2024-11-09T00:23:09,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111849985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111849985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111849994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111849994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111849994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:10,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8367c08317b04774ba84b495fd51b535 2024-11-09T00:23:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/8acfd557ae414761a76141168d1c9b1b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b 2024-11-09T00:23:10,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b, entries=150, sequenceid=131, filesize=30.2 K 2024-11-09T00:23:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/837e6b72eb0d414ca7deead705a43825 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825 2024-11-09T00:23:10,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825, entries=150, sequenceid=131, filesize=11.7 K 2024-11-09T00:23:10,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8367c08317b04774ba84b495fd51b535 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535 2024-11-09T00:23:10,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535, entries=150, sequenceid=131, filesize=11.7 K 2024-11-09T00:23:10,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3016adb28f5d0508edaeae670cdf0011 in 1696ms, sequenceid=131, compaction requested=false 2024-11-09T00:23:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-09T00:23:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-09T00:23:10,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T00:23:10,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-11-09T00:23:10,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.6230 sec 2024-11-09T00:23:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:10,999 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:11,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T00:23:11,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110988be597a62c94a84a22fa72061c5f0a3_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742174_1350 (size=14794) 2024-11-09T00:23:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111851024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111851033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111851035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111851036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111851037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111851154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111851154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111851155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111851364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111851364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111851365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111851365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111851365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:11,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110988be597a62c94a84a22fa72061c5f0a3_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988be597a62c94a84a22fa72061c5f0a3_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:11,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/14614f9260984be1b41f21a99bb91465, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:11,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/14614f9260984be1b41f21a99bb91465 is 175, key is test_row_0/A:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742175_1351 (size=39749) 2024-11-09T00:23:11,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/14614f9260984be1b41f21a99bb91465 2024-11-09T00:23:11,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ce437caee6dc4a8b9b7ed8a643fbb82d is 50, key is test_row_0/B:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742176_1352 (size=12151) 2024-11-09T00:23:11,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ce437caee6dc4a8b9b7ed8a643fbb82d 2024-11-09T00:23:11,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6d72fb9ecbfb4fbda751622bdab0cfe0 is 50, key is test_row_0/C:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742177_1353 (size=12151) 2024-11-09T00:23:11,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6d72fb9ecbfb4fbda751622bdab0cfe0 2024-11-09T00:23:11,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/14614f9260984be1b41f21a99bb91465 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465 2024-11-09T00:23:11,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465, entries=200, sequenceid=153, filesize=38.8 K 2024-11-09T00:23:11,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ce437caee6dc4a8b9b7ed8a643fbb82d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d 2024-11-09T00:23:11,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d, entries=150, sequenceid=153, filesize=11.9 K 2024-11-09T00:23:11,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6d72fb9ecbfb4fbda751622bdab0cfe0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0 2024-11-09T00:23:11,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0, entries=150, sequenceid=153, filesize=11.9 K 2024-11-09T00:23:11,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3016adb28f5d0508edaeae670cdf0011 in 501ms, sequenceid=153, compaction requested=true 2024-11-09T00:23:11,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:11,500 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:11,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:11,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:11,501 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:11,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:11,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:11,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:11,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:11,502 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,502 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=99.5 K 2024-11-09T00:23:11,502 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465] 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:11,502 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6671163193a8420bafec3b50686d4f4f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:11,503 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,503 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/56b6ef8d108f4b988fb5c856d36789a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=35.5 K 2024-11-09T00:23:11,503 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acfd557ae414761a76141168d1c9b1b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731111787606 2024-11-09T00:23:11,503 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b6ef8d108f4b988fb5c856d36789a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:11,503 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14614f9260984be1b41f21a99bb91465, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:11,503 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 837e6b72eb0d414ca7deead705a43825, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731111787606 2024-11-09T00:23:11,504 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ce437caee6dc4a8b9b7ed8a643fbb82d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:11,517 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:11,526 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:11,527 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/4ac1050bf39247eb94687ba745ae04e9 is 50, key is test_row_0/B:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,541 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411095923b74823b14eafaa630674746382f5_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:11,542 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411095923b74823b14eafaa630674746382f5_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:11,543 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095923b74823b14eafaa630674746382f5_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:11,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742178_1354 (size=12493) 2024-11-09T00:23:11,578 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/4ac1050bf39247eb94687ba745ae04e9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4ac1050bf39247eb94687ba745ae04e9 2024-11-09T00:23:11,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742179_1355 (size=4469) 2024-11-09T00:23:11,586 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 4ac1050bf39247eb94687ba745ae04e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:11,586 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:11,586 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=13, startTime=1731111791501; duration=0sec 2024-11-09T00:23:11,586 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:11,586 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:11,586 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:11,588 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:11,588 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:11,588 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,588 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/74bce3936b55441ebe870753600ca3ec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=35.5 K 2024-11-09T00:23:11,588 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bce3936b55441ebe870753600ca3ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731111786402 2024-11-09T00:23:11,588 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8367c08317b04774ba84b495fd51b535, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731111787606 2024-11-09T00:23:11,589 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d72fb9ecbfb4fbda751622bdab0cfe0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:11,598 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:11,598 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/4e7cbceac90549e7862a50e131fb7dec is 50, key is test_row_0/C:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742180_1356 (size=12493) 2024-11-09T00:23:11,621 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/4e7cbceac90549e7862a50e131fb7dec as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4e7cbceac90549e7862a50e131fb7dec 2024-11-09T00:23:11,628 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 4e7cbceac90549e7862a50e131fb7dec(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:11,628 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:11,628 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=13, startTime=1731111791502; duration=0sec 2024-11-09T00:23:11,628 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:11,628 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:11,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:11,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:23:11,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f08480a8c0e44cd9b8be3d9732ec5244_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111791031/Put/seqid=0 2024-11-09T00:23:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742181_1357 (size=14794) 2024-11-09T00:23:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111851714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111851715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111851724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111851725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111851725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-09T00:23:11,835 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:11,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-09T00:23:11,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:11,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:11,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111851835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111851835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111851844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111851844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111851844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:11,988 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#299 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:11,989 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9f8ec4260aef42418639edef775a872c is 175, key is test_row_0/A:col10/1731111788823/Put/seqid=0 2024-11-09T00:23:11,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:11,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:11,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742182_1358 (size=31447) 2024-11-09T00:23:12,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111852054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111852054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111852064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111852065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111852065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:12,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f08480a8c0e44cd9b8be3d9732ec5244_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f08480a8c0e44cd9b8be3d9732ec5244_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:12,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ebb65fa544e846d5a92cbad8090c210e, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:12,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ebb65fa544e846d5a92cbad8090c210e is 175, key is test_row_0/A:col10/1731111791031/Put/seqid=0 2024-11-09T00:23:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742183_1359 (size=39749) 2024-11-09T00:23:12,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:12,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111852374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111852374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111852384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111852385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111852385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,408 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9f8ec4260aef42418639edef775a872c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c 2024-11-09T00:23:12,419 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 9f8ec4260aef42418639edef775a872c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:12,419 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:12,419 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=13, startTime=1731111791500; duration=0sec 2024-11-09T00:23:12,419 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:12,420 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:12,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:12,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ebb65fa544e846d5a92cbad8090c210e 2024-11-09T00:23:12,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1658a60caf214f2ca7dd4dff1e769016 is 50, key is test_row_0/B:col10/1731111791031/Put/seqid=0 2024-11-09T00:23:12,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742184_1360 (size=12151) 2024-11-09T00:23:12,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111852895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111852895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:12,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111852904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111852904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111852905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:12,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1658a60caf214f2ca7dd4dff1e769016 2024-11-09T00:23:12,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/e20b2ccf5b284104aa67e964a2fe8848 is 50, key is test_row_0/C:col10/1731111791031/Put/seqid=0 2024-11-09T00:23:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742185_1361 (size=12151) 2024-11-09T00:23:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:13,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:13,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:13,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:13,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/e20b2ccf5b284104aa67e964a2fe8848 2024-11-09T00:23:13,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/ebb65fa544e846d5a92cbad8090c210e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e 2024-11-09T00:23:13,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e, entries=200, sequenceid=171, filesize=38.8 K 2024-11-09T00:23:13,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1658a60caf214f2ca7dd4dff1e769016 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016 2024-11-09T00:23:13,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T00:23:13,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/e20b2ccf5b284104aa67e964a2fe8848 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848 2024-11-09T00:23:13,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T00:23:13,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3016adb28f5d0508edaeae670cdf0011 in 1684ms, sequenceid=171, compaction requested=false 2024-11-09T00:23:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:13,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-09T00:23:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,368 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T00:23:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110957278b7a12e34fc987ec34eb21d5e5c5_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111791697/Put/seqid=0 2024-11-09T00:23:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742186_1362 (size=12304) 2024-11-09T00:23:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110957278b7a12e34fc987ec34eb21d5e5c5_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957278b7a12e34fc987ec34eb21d5e5c5_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9cf42d90f3804dfd8ca64cafffae75ee, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9cf42d90f3804dfd8ca64cafffae75ee is 175, key is test_row_0/A:col10/1731111791697/Put/seqid=0 2024-11-09T00:23:13,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742187_1363 (size=31105) 2024-11-09T00:23:13,810 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9cf42d90f3804dfd8ca64cafffae75ee 2024-11-09T00:23:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38065ad59ec74031bc2d42e0bea329d3 is 50, key is test_row_0/B:col10/1731111791697/Put/seqid=0 2024-11-09T00:23:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742188_1364 (size=12151) 2024-11-09T00:23:13,830 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38065ad59ec74031bc2d42e0bea329d3 2024-11-09T00:23:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/44ba0a084f234ac7b1411731ec42f79e is 50, key is test_row_0/C:col10/1731111791697/Put/seqid=0 2024-11-09T00:23:13,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742189_1365 (size=12151) 2024-11-09T00:23:13,843 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/44ba0a084f234ac7b1411731ec42f79e 2024-11-09T00:23:13,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/9cf42d90f3804dfd8ca64cafffae75ee as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee 2024-11-09T00:23:13,851 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee, entries=150, sequenceid=192, filesize=30.4 K 2024-11-09T00:23:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38065ad59ec74031bc2d42e0bea329d3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3 2024-11-09T00:23:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T00:23:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/44ba0a084f234ac7b1411731ec42f79e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,861 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T00:23:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,862 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 494ms, sequenceid=192, compaction requested=true 2024-11-09T00:23:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-09T00:23:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T00:23:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-09T00:23:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.0340 sec 2024-11-09T00:23:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-09T00:23:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:13,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:14,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:14,067 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:14,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094826f811da1744298a9b38f3807bbeb4_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:14,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742190_1366 (size=19774) 2024-11-09T00:23:14,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094826f811da1744298a9b38f3807bbeb4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094826f811da1744298a9b38f3807bbeb4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:14,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4d4caae3590c47c0b7fc27ea3b4a7fdd, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4d4caae3590c47c0b7fc27ea3b4a7fdd is 175, key is test_row_0/A:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111854135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111854137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742191_1367 (size=57033) 2024-11-09T00:23:14,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111854138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4d4caae3590c47c0b7fc27ea3b4a7fdd 2024-11-09T00:23:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111854148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111854150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/814b5c16634d4bd2bc01a07a2bf728f9 is 50, key is test_row_0/B:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742192_1368 (size=12151) 2024-11-09T00:23:14,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/814b5c16634d4bd2bc01a07a2bf728f9 2024-11-09T00:23:14,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/950053eb0a544fc5b82a26b9a9bfda64 is 50, key is test_row_0/C:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742193_1369 (size=12151) 2024-11-09T00:23:14,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/950053eb0a544fc5b82a26b9a9bfda64 2024-11-09T00:23:14,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4d4caae3590c47c0b7fc27ea3b4a7fdd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd 2024-11-09T00:23:14,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd, entries=300, sequenceid=204, filesize=55.7 K 2024-11-09T00:23:14,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/814b5c16634d4bd2bc01a07a2bf728f9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9 2024-11-09T00:23:14,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9, entries=150, sequenceid=204, filesize=11.9 K 2024-11-09T00:23:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/950053eb0a544fc5b82a26b9a9bfda64 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64 2024-11-09T00:23:14,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64, entries=150, sequenceid=204, filesize=11.9 K 2024-11-09T00:23:14,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3016adb28f5d0508edaeae670cdf0011 in 170ms, sequenceid=204, compaction requested=true 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:14,237 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:14,237 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:14,239 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:14,239 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:14,239 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:14,239 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,239 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:14,239 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,240 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4ac1050bf39247eb94687ba745ae04e9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.8 K 2024-11-09T00:23:14,240 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=155.6 K 2024-11-09T00:23:14,240 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,240 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd] 2024-11-09T00:23:14,240 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac1050bf39247eb94687ba745ae04e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:14,240 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8ec4260aef42418639edef775a872c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:14,240 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1658a60caf214f2ca7dd4dff1e769016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731111791031 2024-11-09T00:23:14,240 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb65fa544e846d5a92cbad8090c210e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731111791009 2024-11-09T00:23:14,241 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 38065ad59ec74031bc2d42e0bea329d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731111791697 2024-11-09T00:23:14,241 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf42d90f3804dfd8ca64cafffae75ee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731111791697 2024-11-09T00:23:14,241 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 814b5c16634d4bd2bc01a07a2bf728f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794058 2024-11-09T00:23:14,241 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4caae3590c47c0b7fc27ea3b4a7fdd, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794016 2024-11-09T00:23:14,254 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:14,254 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/43af374e39054c67a084b2a1669a59a3 is 50, key is test_row_0/B:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,255 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:14,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:14,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:23:14,273 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d28358e7b8ee47aab546355904cad226_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,275 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d28358e7b8ee47aab546355904cad226_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,276 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d28358e7b8ee47aab546355904cad226_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742194_1370 (size=12629) 2024-11-09T00:23:14,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098f2d00673a5c4833804c008c228914d4_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111794133/Put/seqid=0 2024-11-09T00:23:14,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742196_1372 (size=14794) 2024-11-09T00:23:14,295 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/43af374e39054c67a084b2a1669a59a3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/43af374e39054c67a084b2a1669a59a3 2024-11-09T00:23:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742195_1371 (size=4469) 2024-11-09T00:23:14,305 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 43af374e39054c67a084b2a1669a59a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:14,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:14,305 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=12, startTime=1731111794237; duration=0sec 2024-11-09T00:23:14,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:14,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:14,306 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,306 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#312 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111854289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,307 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/720a115ff99d4ee08f82336c4302e717 is 175, key is test_row_0/A:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,309 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:14,309 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:14,309 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,309 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4e7cbceac90549e7862a50e131fb7dec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.8 K 2024-11-09T00:23:14,310 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7cbceac90549e7862a50e131fb7dec, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731111788821 2024-11-09T00:23:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742197_1373 (size=31583) 2024-11-09T00:23:14,312 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e20b2ccf5b284104aa67e964a2fe8848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731111791031 2024-11-09T00:23:14,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,313 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ba0a084f234ac7b1411731ec42f79e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731111791697 2024-11-09T00:23:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,313 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 950053eb0a544fc5b82a26b9a9bfda64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794058 2024-11-09T00:23:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,324 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/720a115ff99d4ee08f82336c4302e717 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717 2024-11-09T00:23:14,327 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#314 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111854301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,328 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/7446fbba42154f72a228c9cc79e27a72 is 50, key is test_row_0/C:col10/1731111794066/Put/seqid=0 2024-11-09T00:23:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,333 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 720a115ff99d4ee08f82336c4302e717(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:14,333 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:14,333 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=12, startTime=1731111794237; duration=0sec 2024-11-09T00:23:14,333 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:14,333 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111854305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111854308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111854307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742198_1374 (size=12629) 2024-11-09T00:23:14,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111854414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111854435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111854444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111854444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111854446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:14,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111854624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111854644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111854665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111854665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111854665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:14,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411098f2d00673a5c4833804c008c228914d4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098f2d00673a5c4833804c008c228914d4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:14,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a52afcdef0a348928d25d29c7d38998c, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:14,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a52afcdef0a348928d25d29c7d38998c is 175, key is test_row_0/A:col10/1731111794133/Put/seqid=0 2024-11-09T00:23:14,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742199_1375 (size=39749) 2024-11-09T00:23:14,750 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/7446fbba42154f72a228c9cc79e27a72 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7446fbba42154f72a228c9cc79e27a72 2024-11-09T00:23:14,754 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 7446fbba42154f72a228c9cc79e27a72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:14,754 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:14,754 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=12, startTime=1731111794237; duration=0sec 2024-11-09T00:23:14,755 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:14,755 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:14,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:14,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:14,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111854944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111854954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111854984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,104 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a52afcdef0a348928d25d29c7d38998c 2024-11-09T00:23:15,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/d14c1229549e40bfb05d4947aaec499e is 50, key is test_row_0/B:col10/1731111794133/Put/seqid=0 2024-11-09T00:23:15,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742200_1376 (size=12151) 2024-11-09T00:23:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:15,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111855494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111855494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111855504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:15,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/d14c1229549e40bfb05d4947aaec499e 2024-11-09T00:23:15,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8b824df4b01540b9878f9ac8870a2fce is 50, key is test_row_0/C:col10/1731111794133/Put/seqid=0 2024-11-09T00:23:15,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742201_1377 (size=12151) 2024-11-09T00:23:15,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:15,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8b824df4b01540b9878f9ac8870a2fce 2024-11-09T00:23:15,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a52afcdef0a348928d25d29c7d38998c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c 2024-11-09T00:23:15,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c, entries=200, sequenceid=229, filesize=38.8 K 2024-11-09T00:23:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/d14c1229549e40bfb05d4947aaec499e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e 2024-11-09T00:23:15,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T00:23:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/8b824df4b01540b9878f9ac8870a2fce as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce 2024-11-09T00:23:15,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T00:23:15,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 3016adb28f5d0508edaeae670cdf0011 in 1703ms, sequenceid=229, compaction requested=false 2024-11-09T00:23:15,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:15,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T00:23:15,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:15,995 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:23:16,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f88380e89e0b4b42ac85fd71f77cce34_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111794304/Put/seqid=0 2024-11-09T00:23:16,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742202_1378 (size=12304) 2024-11-09T00:23:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f88380e89e0b4b42ac85fd71f77cce34_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f88380e89e0b4b42ac85fd71f77cce34_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/7f736ffa1a3947dd9e3dacd834358235, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:16,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/7f736ffa1a3947dd9e3dacd834358235 is 175, key is test_row_0/A:col10/1731111794304/Put/seqid=0 2024-11-09T00:23:16,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742203_1379 (size=31105) 2024-11-09T00:23:16,060 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/7f736ffa1a3947dd9e3dacd834358235 2024-11-09T00:23:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38c4c5ae027747b491196753c25bf470 is 50, key is test_row_0/B:col10/1731111794304/Put/seqid=0 2024-11-09T00:23:16,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742204_1380 (size=12151) 2024-11-09T00:23:16,091 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38c4c5ae027747b491196753c25bf470 2024-11-09T00:23:16,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/130a89ede9ee4fb390639bacabfe6f50 is 50, key is test_row_0/C:col10/1731111794304/Put/seqid=0 2024-11-09T00:23:16,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742205_1381 (size=12151) 2024-11-09T00:23:16,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/130a89ede9ee4fb390639bacabfe6f50 2024-11-09T00:23:16,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/7f736ffa1a3947dd9e3dacd834358235 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235 2024-11-09T00:23:16,129 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235, entries=150, sequenceid=244, filesize=30.4 K 2024-11-09T00:23:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/38c4c5ae027747b491196753c25bf470 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470 2024-11-09T00:23:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,134 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T00:23:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/130a89ede9ee4fb390639bacabfe6f50 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50 2024-11-09T00:23:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,140 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50, entries=150, sequenceid=244, filesize=11.9 K 2024-11-09T00:23:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 146ms, sequenceid=244, compaction requested=true 2024-11-09T00:23:16,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:16,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:16,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-09T00:23:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-09T00:23:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1430 sec 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1610 sec 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:16,635 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:16,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:16,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110995a5c92550064a5b850d54fa7dc3b205_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742206_1382 (size=12304) 2024-11-09T00:23:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111856715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111856715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111856716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111856717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111856727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111856834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111856834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111856834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111856845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111856845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111857045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111857045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111857045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111857064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111857064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,111 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:17,114 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110995a5c92550064a5b850d54fa7dc3b205_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995a5c92550064a5b850d54fa7dc3b205_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:17,115 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/36c0bcfa65ff4be3ae2d88e92754d906, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:17,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/36c0bcfa65ff4be3ae2d88e92754d906 is 175, key is test_row_0/A:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742207_1383 (size=31105) 2024-11-09T00:23:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111857354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111857365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111857365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111857374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111857374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/36c0bcfa65ff4be3ae2d88e92754d906 2024-11-09T00:23:17,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/e17ac6a0779d4fb9b8309e1d1295d4c8 is 50, key is test_row_0/B:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:17,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742208_1384 (size=12151) 2024-11-09T00:23:17,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/e17ac6a0779d4fb9b8309e1d1295d4c8 2024-11-09T00:23:17,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/4332ac0cd1944389aa052a7d4d48255f is 50, key is test_row_0/C:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:17,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742209_1385 (size=12151) 2024-11-09T00:23:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111857874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111857884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111857884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111857895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111857900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:17,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/4332ac0cd1944389aa052a7d4d48255f 2024-11-09T00:23:17,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/36c0bcfa65ff4be3ae2d88e92754d906 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906 2024-11-09T00:23:17,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906, entries=150, sequenceid=255, filesize=30.4 K 2024-11-09T00:23:17,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/e17ac6a0779d4fb9b8309e1d1295d4c8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8 2024-11-09T00:23:17,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T00:23:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/4332ac0cd1944389aa052a7d4d48255f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f 2024-11-09T00:23:17,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T00:23:17,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3016adb28f5d0508edaeae670cdf0011 in 1326ms, sequenceid=255, compaction requested=true 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:17,962 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:17,962 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:17,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:17,963 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:17,963 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:17,963 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:17,963 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:17,963 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:17,963 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:17,963 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=130.4 K 2024-11-09T00:23:17,963 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/43af374e39054c67a084b2a1669a59a3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.9 K 2024-11-09T00:23:17,963 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:17,963 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906] 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 43af374e39054c67a084b2a1669a59a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794058 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720a115ff99d4ee08f82336c4302e717, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794058 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d14c1229549e40bfb05d4947aaec499e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731111794133 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a52afcdef0a348928d25d29c7d38998c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731111794133 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c4c5ae027747b491196753c25bf470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731111794300 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e17ac6a0779d4fb9b8309e1d1295d4c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:17,964 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f736ffa1a3947dd9e3dacd834358235, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731111794300 2024-11-09T00:23:17,965 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c0bcfa65ff4be3ae2d88e92754d906, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:17,978 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:17,980 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:17,980 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/304c414bb03742d6a406c28e7607e868 is 50, key is test_row_0/B:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:17,984 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e46447668bb044129d58ac048f08032c_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:17,987 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e46447668bb044129d58ac048f08032c_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:17,987 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e46447668bb044129d58ac048f08032c_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:18,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742210_1386 (size=12765) 2024-11-09T00:23:18,015 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/304c414bb03742d6a406c28e7607e868 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/304c414bb03742d6a406c28e7607e868 2024-11-09T00:23:18,027 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 304c414bb03742d6a406c28e7607e868(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:18,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:18,027 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=12, startTime=1731111797962; duration=0sec 2024-11-09T00:23:18,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:18,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:18,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:18,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742211_1387 (size=4469) 2024-11-09T00:23:18,035 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#324 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:18,036 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/27cd2fe5ca7542d5bb758f8f75c2fb24 is 175, key is test_row_0/A:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:18,037 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:18,037 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:18,037 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:18,037 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7446fbba42154f72a228c9cc79e27a72, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=47.9 K 2024-11-09T00:23:18,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7446fbba42154f72a228c9cc79e27a72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731111794058 2024-11-09T00:23:18,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b824df4b01540b9878f9ac8870a2fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731111794133 2024-11-09T00:23:18,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 130a89ede9ee4fb390639bacabfe6f50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731111794300 2024-11-09T00:23:18,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4332ac0cd1944389aa052a7d4d48255f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:18,055 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:18,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/bea048968ed54e2497f9f9bb673e0b52 is 50, key is test_row_0/C:col10/1731111796634/Put/seqid=0 2024-11-09T00:23:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742212_1388 (size=31719) 2024-11-09T00:23:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742213_1389 (size=12765) 2024-11-09T00:23:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T00:23:18,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:18,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T00:23:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:18,167 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:18,168 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:18,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:18,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T00:23:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:18,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:23:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109191dc0162ff5433aa0f663a31acc9480_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111796712/Put/seqid=0 2024-11-09T00:23:18,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742214_1390 (size=12454) 2024-11-09T00:23:18,463 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/27cd2fe5ca7542d5bb758f8f75c2fb24 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24 2024-11-09T00:23:18,467 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 27cd2fe5ca7542d5bb758f8f75c2fb24(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:18,467 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:18,467 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=12, startTime=1731111797962; duration=0sec 2024-11-09T00:23:18,467 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:18,467 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:18,474 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/bea048968ed54e2497f9f9bb673e0b52 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bea048968ed54e2497f9f9bb673e0b52 2024-11-09T00:23:18,478 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into bea048968ed54e2497f9f9bb673e0b52(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:18,478 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:18,478 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=12, startTime=1731111797962; duration=0sec 2024-11-09T00:23:18,478 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:18,479 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:18,741 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109191dc0162ff5433aa0f663a31acc9480_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109191dc0162ff5433aa0f663a31acc9480_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6424559e6c314ba1b78eb8e0a47d2bbd, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6424559e6c314ba1b78eb8e0a47d2bbd is 175, key is test_row_0/A:col10/1731111796712/Put/seqid=0 2024-11-09T00:23:18,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742215_1391 (size=31255) 2024-11-09T00:23:18,755 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6424559e6c314ba1b78eb8e0a47d2bbd 2024-11-09T00:23:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1b76cddf91e5457f9c15931c9fad0b36 is 50, key is test_row_0/B:col10/1731111796712/Put/seqid=0 2024-11-09T00:23:18,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742216_1392 (size=12301) 2024-11-09T00:23:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:18,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111858915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111858915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111858917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:18,922 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 3016adb28f5d0508edaeae670cdf0011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 3016adb28f5d0508edaeae670cdf0011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111858920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:18,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111858922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111859035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111859035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111859035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,168 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1b76cddf91e5457f9c15931c9fad0b36 2024-11-09T00:23:19,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/c0138ca809f34500b51af28d3e42024a is 50, key is test_row_0/C:col10/1731111796712/Put/seqid=0 2024-11-09T00:23:19,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742217_1393 (size=12301) 2024-11-09T00:23:19,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111859255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111859255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111859255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111859564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111859564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,578 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/c0138ca809f34500b51af28d3e42024a 2024-11-09T00:23:19,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111859574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:19,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/6424559e6c314ba1b78eb8e0a47d2bbd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd 2024-11-09T00:23:19,586 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd, entries=150, sequenceid=281, filesize=30.5 K 2024-11-09T00:23:19,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/1b76cddf91e5457f9c15931c9fad0b36 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36 2024-11-09T00:23:19,592 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36, entries=150, sequenceid=281, filesize=12.0 K 2024-11-09T00:23:19,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/c0138ca809f34500b51af28d3e42024a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a 2024-11-09T00:23:19,597 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a, entries=150, sequenceid=281, filesize=12.0 K 2024-11-09T00:23:19,598 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3016adb28f5d0508edaeae670cdf0011 in 1275ms, sequenceid=281, compaction requested=false 2024-11-09T00:23:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T00:23:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T00:23:19,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T00:23:19,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-09T00:23:19,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4400 sec 2024-11-09T00:23:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:20,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:20,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:23:20,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097a47742eb70749b8b35dfc2d9e18e8ac_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:20,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742218_1394 (size=14994) 2024-11-09T00:23:20,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111860143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111860153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111860154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111860264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111860275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111860275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T00:23:20,314 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:20,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-09T00:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:20,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:20,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:20,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:20,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111860484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111860484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111860485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:20,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097a47742eb70749b8b35dfc2d9e18e8ac_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097a47742eb70749b8b35dfc2d9e18e8ac_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:20,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a9d58e97047d4089acd677faf3e350fa, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:20,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a9d58e97047d4089acd677faf3e350fa is 175, key is test_row_0/A:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:20,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742219_1395 (size=39949) 2024-11-09T00:23:20,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:20,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111860804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111860804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111860805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:20,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:20,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a9d58e97047d4089acd677faf3e350fa 2024-11-09T00:23:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1731111860935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,941 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 3016adb28f5d0508edaeae670cdf0011 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:20,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47212 deadline: 1731111860945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:20,953 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 3016adb28f5d0508edaeae670cdf0011 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:20,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/97f326e380e845c9b3f1b4adc435137d is 50, key is test_row_0/B:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:20,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742220_1396 (size=12301) 2024-11-09T00:23:21,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111861324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111861324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:21,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111861324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:21,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/97f326e380e845c9b3f1b4adc435137d 2024-11-09T00:23:21,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/176c9551c23742c4af9c35b1bd07666c is 50, key is test_row_0/C:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:21,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742221_1397 (size=12301) 2024-11-09T00:23:21,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:21,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:21,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/176c9551c23742c4af9c35b1bd07666c 2024-11-09T00:23:21,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/a9d58e97047d4089acd677faf3e350fa as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa 2024-11-09T00:23:21,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa, entries=200, sequenceid=295, filesize=39.0 K 2024-11-09T00:23:21,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/97f326e380e845c9b3f1b4adc435137d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d 2024-11-09T00:23:21,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d, entries=150, sequenceid=295, filesize=12.0 K 2024-11-09T00:23:21,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/176c9551c23742c4af9c35b1bd07666c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c 2024-11-09T00:23:21,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c, entries=150, sequenceid=295, filesize=12.0 K 2024-11-09T00:23:21,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3016adb28f5d0508edaeae670cdf0011 in 1729ms, sequenceid=295, compaction requested=true 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:21,807 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:21,807 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:21,808 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,808 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,808 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=100.5 K 2024-11-09T00:23:21,808 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/304c414bb03742d6a406c28e7607e868, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=36.5 K 2024-11-09T00:23:21,808 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa] 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 304c414bb03742d6a406c28e7607e868, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27cd2fe5ca7542d5bb758f8f75c2fb24, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b76cddf91e5457f9c15931c9fad0b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731111796712 2024-11-09T00:23:21,808 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6424559e6c314ba1b78eb8e0a47d2bbd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731111796712 2024-11-09T00:23:21,809 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f326e380e845c9b3f1b4adc435137d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111798908 2024-11-09T00:23:21,809 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d58e97047d4089acd677faf3e350fa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111798908 2024-11-09T00:23:21,813 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:21,817 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110991322f05a5fc49be9de19a14c4bcef27_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:21,819 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:21,819 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/cf4b1ea51fe04ca28001d3c812f5eb9c is 50, key is test_row_0/B:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:21,819 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110991322f05a5fc49be9de19a14c4bcef27_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:21,819 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110991322f05a5fc49be9de19a14c4bcef27_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:21,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T00:23:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,847 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:23:21,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742223_1399 (size=4469) 2024-11-09T00:23:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c1a7f59e6d994866905477f2fa0533ea_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111800118/Put/seqid=0 2024-11-09T00:23:21,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742222_1398 (size=13017) 2024-11-09T00:23:21,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742224_1400 (size=12454) 2024-11-09T00:23:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,866 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c1a7f59e6d994866905477f2fa0533ea_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1a7f59e6d994866905477f2fa0533ea_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/241852b2a19444dd8402e589bdd5a08a, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/241852b2a19444dd8402e589bdd5a08a is 175, key is test_row_0/A:col10/1731111800118/Put/seqid=0 2024-11-09T00:23:21,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742225_1401 (size=31255) 2024-11-09T00:23:21,871 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/241852b2a19444dd8402e589bdd5a08a 2024-11-09T00:23:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/0de08ec50e034acbac830a079e42d5d8 is 50, key is test_row_0/B:col10/1731111800118/Put/seqid=0 2024-11-09T00:23:21,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742226_1402 (size=12301) 2024-11-09T00:23:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/0de08ec50e034acbac830a079e42d5d8 2024-11-09T00:23:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/081929ac8f6342a681181b2e541fb453 is 50, key is test_row_0/C:col10/1731111800118/Put/seqid=0 2024-11-09T00:23:21,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742227_1403 (size=12301) 2024-11-09T00:23:21,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/081929ac8f6342a681181b2e541fb453 2024-11-09T00:23:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/241852b2a19444dd8402e589bdd5a08a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a 2024-11-09T00:23:21,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a, entries=150, sequenceid=319, filesize=30.5 K 2024-11-09T00:23:21,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/0de08ec50e034acbac830a079e42d5d8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8 2024-11-09T00:23:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/081929ac8f6342a681181b2e541fb453 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453, entries=150, sequenceid=319, filesize=12.0 K 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 79ms, sequenceid=319, compaction requested=true 2024-11-09T00:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6160 sec 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,251 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#332 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,252 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/d318f2508bb74ab7880c07a29fa04150 is 175, key is test_row_0/A:col10/1731111798908/Put/seqid=0 2024-11-09T00:23:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/cf4b1ea51fe04ca28001d3c812f5eb9c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/cf4b1ea51fe04ca28001d3c812f5eb9c 2024-11-09T00:23:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,272 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into cf4b1ea51fe04ca28001d3c812f5eb9c(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,272 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:22,272 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=13, startTime=1731111801807; duration=0sec 2024-11-09T00:23:22,272 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:22,272 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:22,272 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,275 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:22,275 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:22,275 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,275 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bea048968ed54e2497f9f9bb673e0b52, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=48.5 K 2024-11-09T00:23:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,275 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting bea048968ed54e2497f9f9bb673e0b52, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731111796618 2024-11-09T00:23:22,276 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting c0138ca809f34500b51af28d3e42024a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731111796712 2024-11-09T00:23:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,276 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 176c9551c23742c4af9c35b1bd07666c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111798908 2024-11-09T00:23:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,276 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 081929ac8f6342a681181b2e541fb453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731111800118 2024-11-09T00:23:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742228_1404 (size=31971) 2024-11-09T00:23:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,294 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,295 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/50977094868540b2a15e7f2fc3cbc7d8 is 50, key is test_row_0/C:col10/1731111800118/Put/seqid=0 2024-11-09T00:23:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,297 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/d318f2508bb74ab7880c07a29fa04150 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150 2024-11-09T00:23:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into d318f2508bb74ab7880c07a29fa04150(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:22,302 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=13, startTime=1731111801807; duration=0sec 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:22,302 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T00:23:22,465 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,504 INFO [AsyncFSWAL-0-hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b-prefix:ef986b38239e,35861,1731111692607 {}] wal.AbstractFSWAL(1368): Slow sync cost: 145 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39715,DS-832d4b32-28cf-446d-a540-1a19865fc9da,DISK]] 2024-11-09T00:23:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742229_1405 (size=13051) 2024-11-09T00:23:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-09T00:23:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:22,507 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:22,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:22,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fa7ff5130d04a78aaf75c495e6a994f_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742230_1406 (size=22618) 2024-11-09T00:23:22,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fa7ff5130d04a78aaf75c495e6a994f_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fa7ff5130d04a78aaf75c495e6a994f_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:22,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4af298ec6945419e9d803b8d508cc379, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4af298ec6945419e9d803b8d508cc379 is 175, key is test_row_0/A:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742231_1407 (size=66023) 2024-11-09T00:23:22,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4af298ec6945419e9d803b8d508cc379 2024-11-09T00:23:22,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6097ee60d97c4212b07c68869f32374b is 50, key is test_row_0/B:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742232_1408 (size=12301) 2024-11-09T00:23:22,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6097ee60d97c4212b07c68869f32374b 2024-11-09T00:23:22,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/bcd6b070054d4803adb0671ed10d44a3 is 50, key is test_row_0/C:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T00:23:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742233_1409 (size=12301) 2024-11-09T00:23:22,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/bcd6b070054d4803adb0671ed10d44a3 2024-11-09T00:23:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/4af298ec6945419e9d803b8d508cc379 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379 2024-11-09T00:23:22,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379, entries=350, sequenceid=332, filesize=64.5 K 2024-11-09T00:23:22,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/6097ee60d97c4212b07c68869f32374b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b 2024-11-09T00:23:22,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T00:23:22,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/bcd6b070054d4803adb0671ed10d44a3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3 2024-11-09T00:23:22,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T00:23:22,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3016adb28f5d0508edaeae670cdf0011 in 124ms, sequenceid=332, compaction requested=true 2024-11-09T00:23:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:22,690 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:22,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:22,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T00:23:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:22,694 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:22,694 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/A is initiating minor compaction (all files) 2024-11-09T00:23:22,694 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/A in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,694 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=126.2 K 2024-11-09T00:23:22,694 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,694 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379] 2024-11-09T00:23:22,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:22,695 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d318f2508bb74ab7880c07a29fa04150, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111798908 2024-11-09T00:23:22,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T00:23:22,695 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 241852b2a19444dd8402e589bdd5a08a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731111800118 2024-11-09T00:23:22,696 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af298ec6945419e9d803b8d508cc379, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111802504 2024-11-09T00:23:22,713 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,716 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109efc3abf6e98a4795a2c9e5f282b10ef9_3016adb28f5d0508edaeae670cdf0011 store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b49f67dd999c46a8b08d0bf1656189d6_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111802694/Put/seqid=0 2024-11-09T00:23:22,718 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109efc3abf6e98a4795a2c9e5f282b10ef9_3016adb28f5d0508edaeae670cdf0011, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,718 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109efc3abf6e98a4795a2c9e5f282b10ef9_3016adb28f5d0508edaeae670cdf0011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111862714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111862723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742234_1410 (size=4469) 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,728 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#A#compaction#341 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:22,730 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/90c65428adde4ca0b0018f2b69ce568c is 175, key is test_row_0/A:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742235_1411 (size=12454) 2024-11-09T00:23:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111862727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:22,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b49f67dd999c46a8b08d0bf1656189d6_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b49f67dd999c46a8b08d0bf1656189d6_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:22,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e57dfcc9a1c74250be9bf787c7a251aa, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:22,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e57dfcc9a1c74250be9bf787c7a251aa is 175, key is test_row_0/A:col10/1731111802694/Put/seqid=0 2024-11-09T00:23:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742236_1412 (size=32073) 2024-11-09T00:23:22,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742237_1413 (size=31255) 2024-11-09T00:23:22,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e57dfcc9a1c74250be9bf787c7a251aa 2024-11-09T00:23:22,777 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/90c65428adde4ca0b0018f2b69ce568c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/90c65428adde4ca0b0018f2b69ce568c 2024-11-09T00:23:22,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/fd4edb78622245b6818308a0aa0a0a94 is 50, key is test_row_0/B:col10/1731111802694/Put/seqid=0 2024-11-09T00:23:22,785 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/A of 3016adb28f5d0508edaeae670cdf0011 into 90c65428adde4ca0b0018f2b69ce568c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:22,785 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:22,785 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/A, priority=13, startTime=1731111802690; duration=0sec 2024-11-09T00:23:22,785 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T00:23:22,785 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:22,785 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-09T00:23:22,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T00:23:22,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T00:23:22,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. because compaction request was cancelled 2024-11-09T00:23:22,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:22,786 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:22,789 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:22,789 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/B is initiating minor compaction (all files) 2024-11-09T00:23:22,789 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/B in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,790 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/cf4b1ea51fe04ca28001d3c812f5eb9c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=36.7 K 2024-11-09T00:23:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742238_1414 (size=12301) 2024-11-09T00:23:22,794 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf4b1ea51fe04ca28001d3c812f5eb9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731111798908 2024-11-09T00:23:22,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/fd4edb78622245b6818308a0aa0a0a94 2024-11-09T00:23:22,795 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de08ec50e034acbac830a079e42d5d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731111800118 2024-11-09T00:23:22,795 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6097ee60d97c4212b07c68869f32374b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111802504 2024-11-09T00:23:22,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/cd1f08a4356e4564ae457c0eafc93ed4 is 50, key is test_row_0/C:col10/1731111802694/Put/seqid=0 2024-11-09T00:23:22,801 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:22,802 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/05195571f1334fe49b71d03513bffe73 is 50, key is test_row_0/B:col10/1731111802561/Put/seqid=0 2024-11-09T00:23:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742239_1415 (size=13119) 2024-11-09T00:23:22,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T00:23:22,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:22,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:22,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742240_1416 (size=12301) 2024-11-09T00:23:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111862835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111862835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111862844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:22,910 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/50977094868540b2a15e7f2fc3cbc7d8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/50977094868540b2a15e7f2fc3cbc7d8 2024-11-09T00:23:22,916 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 50977094868540b2a15e7f2fc3cbc7d8(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:22,916 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:22,916 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=12, startTime=1731111801807; duration=0sec 2024-11-09T00:23:22,916 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:22,916 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:22,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T00:23:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:22,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47230 deadline: 1731111863045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731111863045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:23,052 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-11-09T00:23:23,052 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,052 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,052 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,052 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-11-09T00:23:23,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,052 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,052 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,052 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:23,057 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-11-09T00:23:23,058 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,058 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,058 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,060 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-11-09T00:23:23,060 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,060 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,060 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,061 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-11-09T00:23:23,061 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,061 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,061 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47216 deadline: 1731111863064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:23:23,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T00:23:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:23,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:23,219 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/05195571f1334fe49b71d03513bffe73 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/05195571f1334fe49b71d03513bffe73 2024-11-09T00:23:23,224 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/B of 3016adb28f5d0508edaeae670cdf0011 into 05195571f1334fe49b71d03513bffe73(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:23,224 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:23,224 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/B, priority=13, startTime=1731111802693; duration=0sec 2024-11-09T00:23:23,224 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:23,224 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:23,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/cd1f08a4356e4564ae457c0eafc93ed4 2024-11-09T00:23:23,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/e57dfcc9a1c74250be9bf787c7a251aa as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e57dfcc9a1c74250be9bf787c7a251aa 2024-11-09T00:23:23,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e57dfcc9a1c74250be9bf787c7a251aa, entries=150, sequenceid=359, filesize=30.5 K 2024-11-09T00:23:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/fd4edb78622245b6818308a0aa0a0a94 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/fd4edb78622245b6818308a0aa0a0a94 2024-11-09T00:23:23,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/fd4edb78622245b6818308a0aa0a0a94, entries=150, sequenceid=359, filesize=12.0 K 2024-11-09T00:23:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/cd1f08a4356e4564ae457c0eafc93ed4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4 2024-11-09T00:23:23,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4, entries=150, sequenceid=359, filesize=12.0 K 2024-11-09T00:23:23,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 3016adb28f5d0508edaeae670cdf0011 in 549ms, sequenceid=359, compaction requested=true 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:23,244 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:23,244 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3016adb28f5d0508edaeae670cdf0011:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:23,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. because compaction request was cancelled 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. because compaction request was cancelled 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:B 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:A 2024-11-09T00:23:23,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:23,246 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:23,246 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 3016adb28f5d0508edaeae670cdf0011/C is initiating minor compaction (all files) 2024-11-09T00:23:23,246 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3016adb28f5d0508edaeae670cdf0011/C in TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:23,246 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/50977094868540b2a15e7f2fc3cbc7d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp, totalSize=36.8 K 2024-11-09T00:23:23,246 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50977094868540b2a15e7f2fc3cbc7d8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731111800118 2024-11-09T00:23:23,246 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd6b070054d4803adb0671ed10d44a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111802504 2024-11-09T00:23:23,247 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1f08a4356e4564ae457c0eafc93ed4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731111802638 2024-11-09T00:23:23,252 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3016adb28f5d0508edaeae670cdf0011#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:23,253 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/64e10fbda79948719d8f47b5da189631 is 50, key is test_row_0/C:col10/1731111802694/Put/seqid=0 2024-11-09T00:23:23,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742241_1417 (size=13153) 2024-11-09T00:23:23,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T00:23:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:23,277 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T00:23:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109bac6db3e3ad54f2f8c77be943a3974e6_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_1/A:col10/1731111802725/Put/seqid=0 2024-11-09T00:23:23,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742242_1418 (size=7374) 2024-11-09T00:23:23,356 INFO [Thread-1442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1442. 2024-11-09T00:23:23,356 DEBUG [Thread-1442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,356 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,356 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:23,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. as already flushing 2024-11-09T00:23:23,366 INFO [Thread-1438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1438. 2024-11-09T00:23:23,366 DEBUG [Thread-1438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,366 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,366 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,376 INFO [Thread-1444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1444. 2024-11-09T00:23:23,376 DEBUG [Thread-1444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:23,376 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,376 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:23,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:23,661 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/64e10fbda79948719d8f47b5da189631 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/64e10fbda79948719d8f47b5da189631 2024-11-09T00:23:23,664 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3016adb28f5d0508edaeae670cdf0011/C of 3016adb28f5d0508edaeae670cdf0011 into 64e10fbda79948719d8f47b5da189631(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:23,664 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:23,664 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., storeName=3016adb28f5d0508edaeae670cdf0011/C, priority=13, startTime=1731111803244; duration=0sec 2024-11-09T00:23:23,665 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:23,665 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3016adb28f5d0508edaeae670cdf0011:C 2024-11-09T00:23:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:23,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109bac6db3e3ad54f2f8c77be943a3974e6_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bac6db3e3ad54f2f8c77be943a3974e6_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/88d17a2858764a5995067aa385ff6263, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:23,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/88d17a2858764a5995067aa385ff6263 is 175, key is test_row_1/A:col10/1731111802725/Put/seqid=0 2024-11-09T00:23:23,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742243_1419 (size=13865) 2024-11-09T00:23:24,095 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/88d17a2858764a5995067aa385ff6263 2024-11-09T00:23:24,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ca77149b916347d0a3bbb6be1aa0b7d9 is 50, key is test_row_1/B:col10/1731111802725/Put/seqid=0 2024-11-09T00:23:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742244_1420 (size=7415) 2024-11-09T00:23:24,510 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ca77149b916347d0a3bbb6be1aa0b7d9 2024-11-09T00:23:24,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6747c30ea76f4a63b1c874ded6911df6 is 50, key is test_row_1/C:col10/1731111802725/Put/seqid=0 2024-11-09T00:23:24,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742245_1421 (size=7415) 2024-11-09T00:23:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:24,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6747c30ea76f4a63b1c874ded6911df6 2024-11-09T00:23:24,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/88d17a2858764a5995067aa385ff6263 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/88d17a2858764a5995067aa385ff6263 2024-11-09T00:23:24,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/88d17a2858764a5995067aa385ff6263, entries=50, sequenceid=371, filesize=13.5 K 2024-11-09T00:23:24,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/ca77149b916347d0a3bbb6be1aa0b7d9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ca77149b916347d0a3bbb6be1aa0b7d9 2024-11-09T00:23:24,933 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ca77149b916347d0a3bbb6be1aa0b7d9, entries=50, sequenceid=371, filesize=7.2 K 2024-11-09T00:23:24,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/6747c30ea76f4a63b1c874ded6911df6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6747c30ea76f4a63b1c874ded6911df6 2024-11-09T00:23:24,937 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6747c30ea76f4a63b1c874ded6911df6, entries=50, sequenceid=371, filesize=7.2 K 2024-11-09T00:23:24,938 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=20.13 KB/20610 for 3016adb28f5d0508edaeae670cdf0011 in 1661ms, sequenceid=371, compaction requested=true 2024-11-09T00:23:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 3016adb28f5d0508edaeae670cdf0011: 2024-11-09T00:23:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-09T00:23:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-09T00:23:24,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T00:23:24,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4300 sec 2024-11-09T00:23:24,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4750 sec 2024-11-09T00:23:24,956 INFO [Thread-1440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1440. 2024-11-09T00:23:24,956 DEBUG [Thread-1440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:24,956 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:24,956 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:24,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:24,985 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-11-09T00:23:24,985 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:24,985 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:24,985 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:24,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T00:23:26,665 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1073 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3219 rows 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1076 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3228 rows 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1068 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3203 rows 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1070 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3207 rows 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-09T00:23:26,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3230 rows 2024-11-09T00:23:26,665 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:23:26,665 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:23:26,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:26,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:26,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:26,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:23:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:26,669 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111806669"}]},"ts":"1731111806669"} 2024-11-09T00:23:26,671 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:23:26,671 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:23:26,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:23:26,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, UNASSIGN}] 2024-11-09T00:23:26,674 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, UNASSIGN 2024-11-09T00:23:26,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:26,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, UNASSIGN because future has completed 2024-11-09T00:23:26,676 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:23:26,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:26,828 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:26,828 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:23:26,828 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 3016adb28f5d0508edaeae670cdf0011, disabling compactions & flushes 2024-11-09T00:23:26,829 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:26,829 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:26,829 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. after waiting 0 ms 2024-11-09T00:23:26,829 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:26,829 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 3016adb28f5d0508edaeae670cdf0011 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T00:23:26,835 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097592646294194e62a401c935dd354c63_3016adb28f5d0508edaeae670cdf0011 is 50, key is test_row_0/A:col10/1731111804984/Put/seqid=0 2024-11-09T00:23:26,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742246_1422 (size=12454) 2024-11-09T00:23:26,839 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:26,842 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097592646294194e62a401c935dd354c63_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097592646294194e62a401c935dd354c63_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:26,843 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/fa30b0260f684c8f887a118eb4dcfa09, store: [table=TestAcidGuarantees family=A region=3016adb28f5d0508edaeae670cdf0011] 2024-11-09T00:23:26,843 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/fa30b0260f684c8f887a118eb4dcfa09 is 175, key is test_row_0/A:col10/1731111804984/Put/seqid=0 2024-11-09T00:23:26,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742247_1423 (size=31255) 2024-11-09T00:23:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:27,248 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/fa30b0260f684c8f887a118eb4dcfa09 2024-11-09T00:23:27,256 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7efbd83ed9534bb5a6f3b64422cc1c66 is 50, key is test_row_0/B:col10/1731111804984/Put/seqid=0 2024-11-09T00:23:27,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742248_1424 (size=12301) 2024-11-09T00:23:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:27,660 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7efbd83ed9534bb5a6f3b64422cc1c66 2024-11-09T00:23:27,669 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/f1b5d601ecb445b2b02f33b048c394a1 is 50, key is test_row_0/C:col10/1731111804984/Put/seqid=0 2024-11-09T00:23:27,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742249_1425 (size=12301) 2024-11-09T00:23:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:28,074 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/f1b5d601ecb445b2b02f33b048c394a1 2024-11-09T00:23:28,079 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/A/fa30b0260f684c8f887a118eb4dcfa09 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/fa30b0260f684c8f887a118eb4dcfa09 2024-11-09T00:23:28,084 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/fa30b0260f684c8f887a118eb4dcfa09, entries=150, sequenceid=380, filesize=30.5 K 2024-11-09T00:23:28,085 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/B/7efbd83ed9534bb5a6f3b64422cc1c66 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7efbd83ed9534bb5a6f3b64422cc1c66 2024-11-09T00:23:28,089 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7efbd83ed9534bb5a6f3b64422cc1c66, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T00:23:28,090 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/.tmp/C/f1b5d601ecb445b2b02f33b048c394a1 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/f1b5d601ecb445b2b02f33b048c394a1 2024-11-09T00:23:28,094 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/f1b5d601ecb445b2b02f33b048c394a1, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T00:23:28,095 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 1266ms, sequenceid=380, compaction requested=true 2024-11-09T00:23:28,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379] to archive 2024-11-09T00:23:28,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:28,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8ee40aae6d724dc8b2de12b15e98ecd2 2024-11-09T00:23:28,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/923ba80ab5e84f4098096409ac0ad176 2024-11-09T00:23:28,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/031c5bea09f3447ba8e692cf5342cf89 2024-11-09T00:23:28,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/2f490e487b9a4bda8c616e4ee01029ec 2024-11-09T00:23:28,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ab7bbb6ea1f3478d9f8602447c1d8c1d 2024-11-09T00:23:28,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/941808545a5a4858a0b6fb43cfcaf52b 2024-11-09T00:23:28,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e060d035fcd34b65817e5523374b5c1e 2024-11-09T00:23:28,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6671163193a8420bafec3b50686d4f4f 2024-11-09T00:23:28,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/8acfd557ae414761a76141168d1c9b1b 2024-11-09T00:23:28,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/14614f9260984be1b41f21a99bb91465 2024-11-09T00:23:28,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9f8ec4260aef42418639edef775a872c 2024-11-09T00:23:28,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/ebb65fa544e846d5a92cbad8090c210e 2024-11-09T00:23:28,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/9cf42d90f3804dfd8ca64cafffae75ee 2024-11-09T00:23:28,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4d4caae3590c47c0b7fc27ea3b4a7fdd 2024-11-09T00:23:28,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/720a115ff99d4ee08f82336c4302e717 2024-11-09T00:23:28,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a52afcdef0a348928d25d29c7d38998c 2024-11-09T00:23:28,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/7f736ffa1a3947dd9e3dacd834358235 2024-11-09T00:23:28,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/27cd2fe5ca7542d5bb758f8f75c2fb24 2024-11-09T00:23:28,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/36c0bcfa65ff4be3ae2d88e92754d906 2024-11-09T00:23:28,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/6424559e6c314ba1b78eb8e0a47d2bbd 2024-11-09T00:23:28,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/a9d58e97047d4089acd677faf3e350fa 2024-11-09T00:23:28,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/d318f2508bb74ab7880c07a29fa04150 2024-11-09T00:23:28,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/241852b2a19444dd8402e589bdd5a08a 2024-11-09T00:23:28,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/4af298ec6945419e9d803b8d508cc379 2024-11-09T00:23:28,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/685f658e07c9473b869cd447db031d55, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/56b6ef8d108f4b988fb5c856d36789a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4ac1050bf39247eb94687ba745ae04e9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/43af374e39054c67a084b2a1669a59a3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/304c414bb03742d6a406c28e7607e868, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/cf4b1ea51fe04ca28001d3c812f5eb9c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b] to archive 2024-11-09T00:23:28,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:28,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7a94d54ab5e74ad19b47f7d29eecf1d4 2024-11-09T00:23:28,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/a8b507e824354fdda70ffc3b56692623 2024-11-09T00:23:28,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6c6e98d6aa994e71810608e59e8e664a 2024-11-09T00:23:28,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/685f658e07c9473b869cd447db031d55 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/685f658e07c9473b869cd447db031d55 2024-11-09T00:23:28,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/b2b987123211446e8c7d11050256c254 2024-11-09T00:23:28,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ec57328f519548efa153e52d14332f72 2024-11-09T00:23:28,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/56b6ef8d108f4b988fb5c856d36789a6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/56b6ef8d108f4b988fb5c856d36789a6 2024-11-09T00:23:28,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4bcea2af383949b6a6def5c010b4e944 2024-11-09T00:23:28,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/837e6b72eb0d414ca7deead705a43825 2024-11-09T00:23:28,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4ac1050bf39247eb94687ba745ae04e9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/4ac1050bf39247eb94687ba745ae04e9 2024-11-09T00:23:28,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ce437caee6dc4a8b9b7ed8a643fbb82d 2024-11-09T00:23:28,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1658a60caf214f2ca7dd4dff1e769016 2024-11-09T00:23:28,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38065ad59ec74031bc2d42e0bea329d3 2024-11-09T00:23:28,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/43af374e39054c67a084b2a1669a59a3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/43af374e39054c67a084b2a1669a59a3 2024-11-09T00:23:28,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/814b5c16634d4bd2bc01a07a2bf728f9 2024-11-09T00:23:28,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/d14c1229549e40bfb05d4947aaec499e 2024-11-09T00:23:28,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/38c4c5ae027747b491196753c25bf470 2024-11-09T00:23:28,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/304c414bb03742d6a406c28e7607e868 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/304c414bb03742d6a406c28e7607e868 2024-11-09T00:23:28,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/e17ac6a0779d4fb9b8309e1d1295d4c8 2024-11-09T00:23:28,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/1b76cddf91e5457f9c15931c9fad0b36 2024-11-09T00:23:28,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/cf4b1ea51fe04ca28001d3c812f5eb9c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/cf4b1ea51fe04ca28001d3c812f5eb9c 2024-11-09T00:23:28,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/97f326e380e845c9b3f1b4adc435137d 2024-11-09T00:23:28,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/0de08ec50e034acbac830a079e42d5d8 2024-11-09T00:23:28,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/6097ee60d97c4212b07c68869f32374b 2024-11-09T00:23:28,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5800b8f0f08b446aaed758d38fc80209, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/74bce3936b55441ebe870753600ca3ec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4e7cbceac90549e7862a50e131fb7dec, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7446fbba42154f72a228c9cc79e27a72, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bea048968ed54e2497f9f9bb673e0b52, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/50977094868540b2a15e7f2fc3cbc7d8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4] to archive 2024-11-09T00:23:28,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:28,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8357138838fc41789e38922945720b0c 2024-11-09T00:23:28,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7dcfb39106754a618eb5da1ad7c4e68b 2024-11-09T00:23:28,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5800b8f0f08b446aaed758d38fc80209 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5800b8f0f08b446aaed758d38fc80209 2024-11-09T00:23:28,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/30f7e8dff8d24ddaa5392cb6e5d32dfd 2024-11-09T00:23:28,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/ceb3b25165fb4a2ca6503f324f464ab7 2024-11-09T00:23:28,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8d19738adbd64ce0a78b42dc6239de4d 2024-11-09T00:23:28,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/74bce3936b55441ebe870753600ca3ec to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/74bce3936b55441ebe870753600ca3ec 2024-11-09T00:23:28,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/5c71ae266145403eb7e041b65b566bf6 2024-11-09T00:23:28,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8367c08317b04774ba84b495fd51b535 2024-11-09T00:23:28,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4e7cbceac90549e7862a50e131fb7dec to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4e7cbceac90549e7862a50e131fb7dec 2024-11-09T00:23:28,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6d72fb9ecbfb4fbda751622bdab0cfe0 2024-11-09T00:23:28,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/e20b2ccf5b284104aa67e964a2fe8848 2024-11-09T00:23:28,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/44ba0a084f234ac7b1411731ec42f79e 2024-11-09T00:23:28,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7446fbba42154f72a228c9cc79e27a72 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/7446fbba42154f72a228c9cc79e27a72 2024-11-09T00:23:28,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/950053eb0a544fc5b82a26b9a9bfda64 2024-11-09T00:23:28,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/8b824df4b01540b9878f9ac8870a2fce 2024-11-09T00:23:28,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/130a89ede9ee4fb390639bacabfe6f50 2024-11-09T00:23:28,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bea048968ed54e2497f9f9bb673e0b52 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bea048968ed54e2497f9f9bb673e0b52 2024-11-09T00:23:28,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/4332ac0cd1944389aa052a7d4d48255f 2024-11-09T00:23:28,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/c0138ca809f34500b51af28d3e42024a 2024-11-09T00:23:28,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/176c9551c23742c4af9c35b1bd07666c 2024-11-09T00:23:28,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/50977094868540b2a15e7f2fc3cbc7d8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/50977094868540b2a15e7f2fc3cbc7d8 2024-11-09T00:23:28,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/081929ac8f6342a681181b2e541fb453 2024-11-09T00:23:28,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/bcd6b070054d4803adb0671ed10d44a3 2024-11-09T00:23:28,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/cd1f08a4356e4564ae457c0eafc93ed4 2024-11-09T00:23:28,169 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=4 2024-11-09T00:23:28,170 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. 2024-11-09T00:23:28,170 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 3016adb28f5d0508edaeae670cdf0011: Waiting for close lock at 1731111806828Running coprocessor pre-close hooks at 1731111806828Disabling compacts and flushes for region at 1731111806828Disabling writes for close at 1731111806829 (+1 ms)Obtaining lock to block concurrent updates at 1731111806829Preparing flush snapshotting stores in 3016adb28f5d0508edaeae670cdf0011 at 1731111806829Finished memstore snapshotting TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731111806829Flushing stores of TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011. at 1731111806830 (+1 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/A: creating writer at 1731111806830Flushing mob file 3016adb28f5d0508edaeae670cdf0011/A: appending metadata at 1731111806835 (+5 ms)Flushing mob file 3016adb28f5d0508edaeae670cdf0011/A: closing flushed file at 1731111806835Flushing 3016adb28f5d0508edaeae670cdf0011/A: appending metadata at 1731111806843 (+8 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/A: closing flushed file at 1731111806843Flushing 3016adb28f5d0508edaeae670cdf0011/B: creating writer at 1731111807253 (+410 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/B: appending metadata at 1731111807256 (+3 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/B: closing flushed file at 1731111807256Flushing 3016adb28f5d0508edaeae670cdf0011/C: creating writer at 1731111807665 (+409 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/C: appending metadata at 1731111807668 (+3 ms)Flushing 3016adb28f5d0508edaeae670cdf0011/C: closing flushed file at 1731111807668Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68fc7e7c: reopening flushed file at 1731111808078 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78e7e488: reopening flushed file at 1731111808084 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26d4c034: reopening flushed file at 1731111808089 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3016adb28f5d0508edaeae670cdf0011 in 1266ms, sequenceid=380, compaction requested=true at 1731111808095 (+6 ms)Writing region close event to WAL at 1731111808166 (+71 ms)Running coprocessor post-close hooks at 1731111808170 (+4 ms)Closed at 1731111808170 2024-11-09T00:23:28,172 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,172 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=3016adb28f5d0508edaeae670cdf0011, regionState=CLOSED 2024-11-09T00:23:28,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:28,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T00:23:28,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 3016adb28f5d0508edaeae670cdf0011, server=ef986b38239e,35861,1731111692607 in 1.4990 sec 2024-11-09T00:23:28,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-09T00:23:28,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3016adb28f5d0508edaeae670cdf0011, UNASSIGN in 1.5040 sec 2024-11-09T00:23:28,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-09T00:23:28,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-09T00:23:28,182 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111808182"}]},"ts":"1731111808182"} 2024-11-09T00:23:28,183 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:23:28,183 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:23:28,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5180 sec 2024-11-09T00:23:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T00:23:28,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:28,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:23:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T00:23:28,829 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,832 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,835 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits] 2024-11-09T00:23:28,838 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/88d17a2858764a5995067aa385ff6263 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/88d17a2858764a5995067aa385ff6263 2024-11-09T00:23:28,840 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/90c65428adde4ca0b0018f2b69ce568c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/90c65428adde4ca0b0018f2b69ce568c 2024-11-09T00:23:28,841 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e57dfcc9a1c74250be9bf787c7a251aa to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/e57dfcc9a1c74250be9bf787c7a251aa 2024-11-09T00:23:28,842 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/fa30b0260f684c8f887a118eb4dcfa09 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/A/fa30b0260f684c8f887a118eb4dcfa09 2024-11-09T00:23:28,844 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/05195571f1334fe49b71d03513bffe73 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/05195571f1334fe49b71d03513bffe73 2024-11-09T00:23:28,846 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7efbd83ed9534bb5a6f3b64422cc1c66 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/7efbd83ed9534bb5a6f3b64422cc1c66 2024-11-09T00:23:28,847 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ca77149b916347d0a3bbb6be1aa0b7d9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/ca77149b916347d0a3bbb6be1aa0b7d9 2024-11-09T00:23:28,848 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/fd4edb78622245b6818308a0aa0a0a94 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/B/fd4edb78622245b6818308a0aa0a0a94 2024-11-09T00:23:28,850 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/64e10fbda79948719d8f47b5da189631 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/64e10fbda79948719d8f47b5da189631 2024-11-09T00:23:28,851 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6747c30ea76f4a63b1c874ded6911df6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/6747c30ea76f4a63b1c874ded6911df6 2024-11-09T00:23:28,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/f1b5d601ecb445b2b02f33b048c394a1 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/C/f1b5d601ecb445b2b02f33b048c394a1 2024-11-09T00:23:28,856 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits/383.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011/recovered.edits/383.seqid 2024-11-09T00:23:28,857 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,857 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:23:28,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:23:28,858 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T00:23:28,862 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109191dc0162ff5433aa0f663a31acc9480_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109191dc0162ff5433aa0f663a31acc9480_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092293a6612cad48cbaed7b51a1cffb3d9_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092293a6612cad48cbaed7b51a1cffb3d9_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932ab6f71706541ffb5456f633b1311a4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110932ab6f71706541ffb5456f633b1311a4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c39ad54bfb04c4b98d18ba8458f806b_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c39ad54bfb04c4b98d18ba8458f806b_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,866 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094826f811da1744298a9b38f3807bbeb4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094826f811da1744298a9b38f3807bbeb4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094d12e2e03b6345ef90beb0b565bc4770_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094d12e2e03b6345ef90beb0b565bc4770_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fa7ff5130d04a78aaf75c495e6a994f_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fa7ff5130d04a78aaf75c495e6a994f_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,869 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954ba77af54f746f18c13d4f772fb8df9_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110954ba77af54f746f18c13d4f772fb8df9_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,870 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957278b7a12e34fc987ec34eb21d5e5c5_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957278b7a12e34fc987ec34eb21d5e5c5_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097592646294194e62a401c935dd354c63_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097592646294194e62a401c935dd354c63_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097a47742eb70749b8b35dfc2d9e18e8ac_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097a47742eb70749b8b35dfc2d9e18e8ac_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,873 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988be597a62c94a84a22fa72061c5f0a3_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110988be597a62c94a84a22fa72061c5f0a3_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098f2d00673a5c4833804c008c228914d4_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411098f2d00673a5c4833804c008c228914d4_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995a5c92550064a5b850d54fa7dc3b205_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995a5c92550064a5b850d54fa7dc3b205_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b49f67dd999c46a8b08d0bf1656189d6_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b49f67dd999c46a8b08d0bf1656189d6_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,876 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bac6db3e3ad54f2f8c77be943a3974e6_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bac6db3e3ad54f2f8c77be943a3974e6_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,878 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1a7f59e6d994866905477f2fa0533ea_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c1a7f59e6d994866905477f2fa0533ea_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d4c27804ab4c49eb8d0fe08dfa525181_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d4c27804ab4c49eb8d0fe08dfa525181_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,880 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e46ca644148047f7ae9a59264e2b8325_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e46ca644148047f7ae9a59264e2b8325_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,881 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f08480a8c0e44cd9b8be3d9732ec5244_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f08480a8c0e44cd9b8be3d9732ec5244_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f88380e89e0b4b42ac85fd71f77cce34_3016adb28f5d0508edaeae670cdf0011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f88380e89e0b4b42ac85fd71f77cce34_3016adb28f5d0508edaeae670cdf0011 2024-11-09T00:23:28,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:23:28,884 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,887 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:23:28,889 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:23:28,890 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,890 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:23:28,890 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111808890"}]},"ts":"9223372036854775807"} 2024-11-09T00:23:28,893 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:23:28,893 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3016adb28f5d0508edaeae670cdf0011, NAME => 'TestAcidGuarantees,,1731111779898.3016adb28f5d0508edaeae670cdf0011.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:23:28,893 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:23:28,893 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111808893"}]},"ts":"9223372036854775807"} 2024-11-09T00:23:28,895 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:23:28,896 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-09T00:23:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T00:23:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:23:28,935 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:28,947 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 239) - Thread LEAK? -, OpenFileDescriptor=463 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=612 (was 682), ProcessCount=11 (was 11), AvailableMemoryMB=3032 (was 3108) 2024-11-09T00:23:28,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=612, ProcessCount=11, AvailableMemoryMB=3031 2024-11-09T00:23:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:23:28,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:23:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:28,959 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:23:28,959 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:28,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-09T00:23:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:28,960 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:23:28,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742250_1426 (size=959) 2024-11-09T00:23:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:29,368 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:23:29,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742251_1427 (size=53) 2024-11-09T00:23:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:29,777 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:29,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2edb930c55ccfc4a4303c1efb2c810a9, disabling compactions & flushes 2024-11-09T00:23:29,778 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:29,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:29,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. after waiting 0 ms 2024-11-09T00:23:29,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:29,778 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:29,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2edb930c55ccfc4a4303c1efb2c810a9: Waiting for close lock at 1731111809777Disabling compacts and flushes for region at 1731111809777Disabling writes for close at 1731111809778 (+1 ms)Writing region close event to WAL at 1731111809778Closed at 1731111809778 2024-11-09T00:23:29,779 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:23:29,779 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111809779"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111809779"}]},"ts":"1731111809779"} 2024-11-09T00:23:29,780 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:23:29,781 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:23:29,781 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111809781"}]},"ts":"1731111809781"} 2024-11-09T00:23:29,782 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:23:29,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, ASSIGN}] 2024-11-09T00:23:29,783 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, ASSIGN 2024-11-09T00:23:29,784 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:23:29,935 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=2edb930c55ccfc4a4303c1efb2c810a9, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:29,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, ASSIGN because future has completed 2024-11-09T00:23:29,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:30,094 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:30,094 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:23:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:30,096 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,096 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:30,096 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,096 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,097 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,099 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2edb930c55ccfc4a4303c1efb2c810a9 columnFamilyName A 2024-11-09T00:23:30,099 DEBUG [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:30,099 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(327): Store=2edb930c55ccfc4a4303c1efb2c810a9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:30,099 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,100 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2edb930c55ccfc4a4303c1efb2c810a9 columnFamilyName B 2024-11-09T00:23:30,101 DEBUG [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:30,101 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(327): Store=2edb930c55ccfc4a4303c1efb2c810a9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:30,101 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,102 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2edb930c55ccfc4a4303c1efb2c810a9 columnFamilyName C 2024-11-09T00:23:30,102 DEBUG [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:30,103 INFO [StoreOpener-2edb930c55ccfc4a4303c1efb2c810a9-1 {}] regionserver.HStore(327): Store=2edb930c55ccfc4a4303c1efb2c810a9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:30,103 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,103 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,104 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,104 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,104 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,105 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:23:30,106 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,108 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:23:30,108 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 2edb930c55ccfc4a4303c1efb2c810a9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65730627, jitterRate=-0.020537331700325012}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:23:30,108 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:30,109 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 2edb930c55ccfc4a4303c1efb2c810a9: Running coprocessor pre-open hook at 1731111810096Writing region info on filesystem at 1731111810096Initializing all the Stores at 1731111810097 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111810097Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111810097Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111810097Cleaning up temporary data from old regions at 1731111810104 (+7 ms)Running coprocessor post-open hooks at 1731111810108 (+4 ms)Region opened successfully at 1731111810109 (+1 ms) 2024-11-09T00:23:30,109 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., pid=114, masterSystemTime=1731111810089 2024-11-09T00:23:30,111 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:30,111 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:30,111 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=2edb930c55ccfc4a4303c1efb2c810a9, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:30,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:30,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T00:23:30,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 in 176 msec 2024-11-09T00:23:30,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-09T00:23:30,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, ASSIGN in 333 msec 2024-11-09T00:23:30,117 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:23:30,117 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111810117"}]},"ts":"1731111810117"} 2024-11-09T00:23:30,118 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:23:30,119 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:23:30,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:23:30,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-09T00:23:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-09T00:23:31,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:31,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,107 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f034e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77294384, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,113 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:23:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:23:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4601ffe6 2024-11-09T00:23:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:23:31,114 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:23:31,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,117 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e62267a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9902c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d67d86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f70889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7039b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d42e405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,128 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acaa7af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,130 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68409562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beed708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fabd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,136 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62aa8ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,137 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533108b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b36609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,141 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65534691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346cf6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,148 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d5ead0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,149 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3cb90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:31,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:31,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee6c0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:31,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e975f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:31,164 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,164 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,164 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,164 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,166 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-09T00:23:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,167 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,167 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:31,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,168 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:31,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,172 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,172 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2] 2024-11-09T00:23:31,184 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:31,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:31,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/b8d28970a3bf4fbf9771b37ba23bb68b is 50, key is test_row_0/A:col10/1731111811178/Put/seqid=0 2024-11-09T00:23:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742252_1428 (size=14341) 2024-11-09T00:23:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111871252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111871262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111871264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111871265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111871266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111871374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111871375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111871384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111871384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111871384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:31,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111871585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111871585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111871594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111871594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111871594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/b8d28970a3bf4fbf9771b37ba23bb68b 2024-11-09T00:23:31,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/17c8bc408130421a9d1aa33fbc2d432b is 50, key is test_row_0/B:col10/1731111811178/Put/seqid=0 2024-11-09T00:23:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742253_1429 (size=12001) 2024-11-09T00:23:31,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:31,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:31,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:31,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111871894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111871895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111871904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111871904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111871904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:31,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:31,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/17c8bc408130421a9d1aa33fbc2d432b 2024-11-09T00:23:32,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2c0f9cc0acaa4a13881be167a9bbde16 is 50, key is test_row_0/C:col10/1731111811178/Put/seqid=0 2024-11-09T00:23:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742254_1430 (size=12001) 2024-11-09T00:23:32,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:32,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:32,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:32,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:32,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111872404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111872405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111872414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111872414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111872414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:32,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2c0f9cc0acaa4a13881be167a9bbde16 2024-11-09T00:23:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/b8d28970a3bf4fbf9771b37ba23bb68b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b 2024-11-09T00:23:32,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b, entries=200, sequenceid=13, filesize=14.0 K 2024-11-09T00:23:32,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/17c8bc408130421a9d1aa33fbc2d432b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b 2024-11-09T00:23:32,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:23:32,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2c0f9cc0acaa4a13881be167a9bbde16 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16 2024-11-09T00:23:32,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T00:23:32,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1305ms, sequenceid=13, compaction requested=false 2024-11-09T00:23:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T00:23:32,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:32,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T00:23:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:32,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:23:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/7118fec1a24e4d71a675cf3de185ab91 is 50, key is test_row_0/A:col10/1731111811238/Put/seqid=0 2024-11-09T00:23:32,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742255_1431 (size=12001) 2024-11-09T00:23:32,957 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/7118fec1a24e4d71a675cf3de185ab91 2024-11-09T00:23:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/2cad788460cc475aace93578ca922fcf is 50, key is test_row_0/B:col10/1731111811238/Put/seqid=0 2024-11-09T00:23:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742256_1432 (size=12001) 2024-11-09T00:23:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:33,367 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/2cad788460cc475aace93578ca922fcf 2024-11-09T00:23:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/06cd3ddd1fd348178b140ac9e1f26abd is 50, key is test_row_0/C:col10/1731111811238/Put/seqid=0 2024-11-09T00:23:33,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742257_1433 (size=12001) 2024-11-09T00:23:33,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111873437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111873435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111873439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,442 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111873442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111873442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111873545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111873545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111873564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111873564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,705 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:23:33,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111873755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111873755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,778 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/06cd3ddd1fd348178b140ac9e1f26abd 2024-11-09T00:23:33,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/7118fec1a24e4d71a675cf3de185ab91 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91 2024-11-09T00:23:33,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111873774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111873774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:33,785 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T00:23:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/2cad788460cc475aace93578ca922fcf as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf 2024-11-09T00:23:33,789 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T00:23:33,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/06cd3ddd1fd348178b140ac9e1f26abd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd 2024-11-09T00:23:33,806 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T00:23:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1259ms, sequenceid=37, compaction requested=false 2024-11-09T00:23:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-09T00:23:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-09T00:23:33,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T00:23:33,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6390 sec 2024-11-09T00:23:33,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6450 sec 2024-11-09T00:23:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:34,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:23:34,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8d48b483332044af989588b173484467 is 50, key is test_row_0/A:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:34,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742258_1434 (size=14341) 2024-11-09T00:23:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111874120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111874129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111874130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111874131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111874234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111874244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111874244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111874244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111874454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111874464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111874464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111874465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8d48b483332044af989588b173484467 2024-11-09T00:23:34,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/85024e08526c4f819c28e16317e736dd is 50, key is test_row_0/B:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742259_1435 (size=12001) 2024-11-09T00:23:34,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111874774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111874774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111874774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111874774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:34,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/85024e08526c4f819c28e16317e736dd 2024-11-09T00:23:34,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/1ef1cdbd09d24044a69e93267873eef0 is 50, key is test_row_0/C:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742260_1436 (size=12001) 2024-11-09T00:23:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111875294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111875294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111875294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111875294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/1ef1cdbd09d24044a69e93267873eef0 2024-11-09T00:23:35,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8d48b483332044af989588b173484467 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467 2024-11-09T00:23:35,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467, entries=200, sequenceid=50, filesize=14.0 K 2024-11-09T00:23:35,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/85024e08526c4f819c28e16317e736dd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd 2024-11-09T00:23:35,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T00:23:35,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/1ef1cdbd09d24044a69e93267873eef0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0 2024-11-09T00:23:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T00:23:35,335 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:35,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:35,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T00:23:35,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1268ms, sequenceid=50, compaction requested=true 2024-11-09T00:23:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-09T00:23:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:35,338 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:35,339 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:35,339 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:35,339 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:35,339 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,340 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=39.7 K 2024-11-09T00:23:35,340 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d28970a3bf4fbf9771b37ba23bb68b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111811176 2024-11-09T00:23:35,340 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:35,340 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:35,340 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:35,340 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,340 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.2 K 2024-11-09T00:23:35,341 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7118fec1a24e4d71a675cf3de185ab91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731111811238 2024-11-09T00:23:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:35,341 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c8bc408130421a9d1aa33fbc2d432b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111811178 2024-11-09T00:23:35,341 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d48b483332044af989588b173484467, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:35,342 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cad788460cc475aace93578ca922fcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731111811238 2024-11-09T00:23:35,342 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:35,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:35,342 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 85024e08526c4f819c28e16317e736dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:35,353 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:35,353 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:35,353 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/cdb077836c424030864a00b78eb786f2 is 50, key is test_row_0/A:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:35,353 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a6c81bdbab864975b23fe931b1cdb520 is 50, key is test_row_0/B:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:35,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742261_1437 (size=12104) 2024-11-09T00:23:35,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742262_1438 (size=12104) 2024-11-09T00:23:35,375 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a6c81bdbab864975b23fe931b1cdb520 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a6c81bdbab864975b23fe931b1cdb520 2024-11-09T00:23:35,382 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into a6c81bdbab864975b23fe931b1cdb520(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:35,382 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:35,382 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111815339; duration=0sec 2024-11-09T00:23:35,382 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:35,382 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:35,382 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:35,383 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:35,383 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:35,383 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,383 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.2 K 2024-11-09T00:23:35,383 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c0f9cc0acaa4a13881be167a9bbde16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731111811178 2024-11-09T00:23:35,383 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cd3ddd1fd348178b140ac9e1f26abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731111811238 2024-11-09T00:23:35,384 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef1cdbd09d24044a69e93267873eef0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:35,391 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:35,391 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/ac4888f42a8d43449e5681b9504ba76e is 50, key is test_row_0/C:col10/1731111813438/Put/seqid=0 2024-11-09T00:23:35,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742263_1439 (size=12104) 2024-11-09T00:23:35,406 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/ac4888f42a8d43449e5681b9504ba76e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ac4888f42a8d43449e5681b9504ba76e 2024-11-09T00:23:35,409 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into ac4888f42a8d43449e5681b9504ba76e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:35,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:35,410 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111815341; duration=0sec 2024-11-09T00:23:35,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:35,410 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:35,470 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:35,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T00:23:35,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/a554c7c5a71344f9ad150009b3893f03 is 50, key is test_row_0/A:col10/1731111815469/Put/seqid=0 2024-11-09T00:23:35,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742264_1440 (size=14341) 2024-11-09T00:23:35,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T00:23:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111875526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111875634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T00:23:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd] to archive 2024-11-09T00:23:35,725 WARN [regionserver/ef986b38239e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-09T00:23:35,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:35,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/17c8bc408130421a9d1aa33fbc2d432b 2024-11-09T00:23:35,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/2cad788460cc475aace93578ca922fcf 2024-11-09T00:23:35,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/85024e08526c4f819c28e16317e736dd 2024-11-09T00:23:35,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0] to archive 2024-11-09T00:23:35,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:35,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2c0f9cc0acaa4a13881be167a9bbde16 2024-11-09T00:23:35,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/06cd3ddd1fd348178b140ac9e1f26abd 2024-11-09T00:23:35,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ef986b38239e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/1ef1cdbd09d24044a69e93267873eef0 2024-11-09T00:23:35,766 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/cdb077836c424030864a00b78eb786f2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cdb077836c424030864a00b78eb786f2 2024-11-09T00:23:35,771 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into cdb077836c424030864a00b78eb786f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:35,771 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:35,771 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111815338; duration=0sec 2024-11-09T00:23:35,771 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:35,771 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:35,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T00:23:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:35,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111875844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:35,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/a554c7c5a71344f9ad150009b3893f03 2024-11-09T00:23:35,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/cc82a7801c3a4046acb2b06636b802c7 is 50, key is test_row_0/B:col10/1731111815469/Put/seqid=0 2024-11-09T00:23:35,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742265_1441 (size=12001) 2024-11-09T00:23:35,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/cc82a7801c3a4046acb2b06636b802c7 2024-11-09T00:23:35,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/9893b290885d403d99b949e23919f0f2 is 50, key is test_row_0/C:col10/1731111815469/Put/seqid=0 2024-11-09T00:23:35,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742266_1442 (size=12001) 2024-11-09T00:23:35,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/9893b290885d403d99b949e23919f0f2 2024-11-09T00:23:35,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/a554c7c5a71344f9ad150009b3893f03 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03 2024-11-09T00:23:35,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03, entries=200, sequenceid=76, filesize=14.0 K 2024-11-09T00:23:35,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/cc82a7801c3a4046acb2b06636b802c7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7 2024-11-09T00:23:35,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T00:23:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/9893b290885d403d99b949e23919f0f2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2 2024-11-09T00:23:35,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T00:23:35,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2edb930c55ccfc4a4303c1efb2c810a9 in 474ms, sequenceid=76, compaction requested=false 2024-11-09T00:23:35,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:35,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T00:23:35,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:35,961 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:23:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/537e9c258d9948489c3daffea67fd011 is 50, key is test_row_0/A:col10/1731111815504/Put/seqid=0 2024-11-09T00:23:35,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742267_1443 (size=12001) 2024-11-09T00:23:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111876292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111876305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111876305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,315 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111876305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,315 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,316 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111876316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,324 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,378 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/537e9c258d9948489c3daffea67fd011 2024-11-09T00:23:36,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/ff53084cccd940fe8aeba15b6b456451 is 50, key is test_row_0/B:col10/1731111815504/Put/seqid=0 2024-11-09T00:23:36,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742268_1444 (size=12001) 2024-11-09T00:23:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111876404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:36,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111876614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:36,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/ff53084cccd940fe8aeba15b6b456451 2024-11-09T00:23:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/508f962164e24d10b70ce6dcfebe5fb7 is 50, key is test_row_0/C:col10/1731111815504/Put/seqid=0 2024-11-09T00:23:36,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742269_1445 (size=12001) 2024-11-09T00:23:36,802 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/508f962164e24d10b70ce6dcfebe5fb7 2024-11-09T00:23:36,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/537e9c258d9948489c3daffea67fd011 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011 2024-11-09T00:23:36,809 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T00:23:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/ff53084cccd940fe8aeba15b6b456451 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451 2024-11-09T00:23:36,813 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T00:23:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/508f962164e24d10b70ce6dcfebe5fb7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7 2024-11-09T00:23:36,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T00:23:36,818 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2edb930c55ccfc4a4303c1efb2c810a9 in 858ms, sequenceid=89, compaction requested=true 2024-11-09T00:23:36,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:36,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:36,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-09T00:23:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-09T00:23:36,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-09T00:23:36,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4770 sec 2024-11-09T00:23:36,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4860 sec 2024-11-09T00:23:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:36,928 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:36,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:23:36,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/42cc8ddee7c5410989d9cc9fac507cf4 is 50, key is test_row_0/A:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:36,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742270_1446 (size=14341) 2024-11-09T00:23:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111877010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111877124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:37,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111877344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:37,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/42cc8ddee7c5410989d9cc9fac507cf4 2024-11-09T00:23:37,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/7217ce12d8b34139af64647f47ce713e is 50, key is test_row_0/B:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742271_1447 (size=12001) 2024-11-09T00:23:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T00:23:37,484 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:37,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-09T00:23:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:37,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:37,487 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:37,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:37,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T00:23:37,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:37,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111877655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:37,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T00:23:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:37,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/7217ce12d8b34139af64647f47ce713e 2024-11-09T00:23:37,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/593c24b49b684ce59750a90719426131 is 50, key is test_row_0/C:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:37,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742272_1448 (size=12001) 2024-11-09T00:23:37,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T00:23:37,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:37,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:38,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T00:23:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:38,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:38,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111878174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/593c24b49b684ce59750a90719426131 2024-11-09T00:23:38,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/42cc8ddee7c5410989d9cc9fac507cf4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4 2024-11-09T00:23:38,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4, entries=200, sequenceid=114, filesize=14.0 K 2024-11-09T00:23:38,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/7217ce12d8b34139af64647f47ce713e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e 2024-11-09T00:23:38,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T00:23:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/593c24b49b684ce59750a90719426131 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131 2024-11-09T00:23:38,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T00:23:38,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1313ms, sequenceid=114, compaction requested=true 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:38,241 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:38,241 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:38,242 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:38,242 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:38,243 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,243 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cdb077836c424030864a00b78eb786f2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=51.5 K 2024-11-09T00:23:38,243 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:38,243 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:38,243 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,243 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb077836c424030864a00b78eb786f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:38,243 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a6c81bdbab864975b23fe931b1cdb520, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=47.0 K 2024-11-09T00:23:38,244 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a554c7c5a71344f9ad150009b3893f03, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731111814128 2024-11-09T00:23:38,244 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c81bdbab864975b23fe931b1cdb520, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:38,244 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting cc82a7801c3a4046acb2b06636b802c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731111814128 2024-11-09T00:23:38,244 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537e9c258d9948489c3daffea67fd011, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111815504 2024-11-09T00:23:38,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ff53084cccd940fe8aeba15b6b456451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111815504 2024-11-09T00:23:38,245 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42cc8ddee7c5410989d9cc9fac507cf4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:38,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7217ce12d8b34139af64647f47ce713e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:38,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-09T00:23:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,251 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:38,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/6714ccd4daea4481882078da2c65ddc9 is 50, key is test_row_0/A:col10/1731111816954/Put/seqid=0 2024-11-09T00:23:38,262 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:38,263 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/ff80e35074944389a344b287cd1e4769 is 50, key is test_row_0/B:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:38,267 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:38,267 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/4d464dec711b413380bd544bc9acd07b is 50, key is test_row_0/A:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:38,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742273_1449 (size=12001) 2024-11-09T00:23:38,270 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/6714ccd4daea4481882078da2c65ddc9 2024-11-09T00:23:38,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/94fbc9d701f54646be10ff65919fe371 is 50, key is test_row_0/B:col10/1731111816954/Put/seqid=0 2024-11-09T00:23:38,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742274_1450 (size=12139) 2024-11-09T00:23:38,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742275_1451 (size=12241) 2024-11-09T00:23:38,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742276_1452 (size=12001) 2024-11-09T00:23:38,297 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/94fbc9d701f54646be10ff65919fe371 2024-11-09T00:23:38,299 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/ff80e35074944389a344b287cd1e4769 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff80e35074944389a344b287cd1e4769 2024-11-09T00:23:38,300 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/4d464dec711b413380bd544bc9acd07b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4d464dec711b413380bd544bc9acd07b 2024-11-09T00:23:38,304 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into ff80e35074944389a344b287cd1e4769(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:38,304 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:38,305 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=12, startTime=1731111818241; duration=0sec 2024-11-09T00:23:38,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:38,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:38,305 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:23:38,306 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into 4d464dec711b413380bd544bc9acd07b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:38,306 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:38,306 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=12, startTime=1731111818241; duration=0sec 2024-11-09T00:23:38,306 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:38,306 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/c077c325038348938ed1b9c126c10a71 is 50, key is test_row_0/C:col10/1731111816954/Put/seqid=0 2024-11-09T00:23:38,310 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:23:38,310 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:38,310 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,310 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ac4888f42a8d43449e5681b9504ba76e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=47.0 K 2024-11-09T00:23:38,310 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4888f42a8d43449e5681b9504ba76e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731111813437 2024-11-09T00:23:38,311 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9893b290885d403d99b949e23919f0f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731111814128 2024-11-09T00:23:38,311 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 508f962164e24d10b70ce6dcfebe5fb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731111815504 2024-11-09T00:23:38,311 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 593c24b49b684ce59750a90719426131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:38,319 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:38,319 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/e220b56d448a4d40bf77d7efaef4d15e is 50, key is test_row_0/C:col10/1731111816927/Put/seqid=0 2024-11-09T00:23:38,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:38,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742277_1453 (size=12001) 2024-11-09T00:23:38,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742278_1454 (size=12139) 2024-11-09T00:23:38,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111878423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111878420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111878431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111878432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111878536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111878536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111878544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111878554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:38,739 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/c077c325038348938ed1b9c126c10a71 2024-11-09T00:23:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/6714ccd4daea4481882078da2c65ddc9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9 2024-11-09T00:23:38,747 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9, entries=150, sequenceid=125, filesize=11.7 K 2024-11-09T00:23:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/94fbc9d701f54646be10ff65919fe371 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371 2024-11-09T00:23:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111878755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111878756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,764 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/e220b56d448a4d40bf77d7efaef4d15e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e220b56d448a4d40bf77d7efaef4d15e 2024-11-09T00:23:38,764 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371, entries=150, sequenceid=125, filesize=11.7 K 2024-11-09T00:23:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/c077c325038348938ed1b9c126c10a71 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71 2024-11-09T00:23:38,769 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into e220b56d448a4d40bf77d7efaef4d15e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:38,769 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:38,769 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=12, startTime=1731111818241; duration=0sec 2024-11-09T00:23:38,769 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:38,769 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:38,769 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71, entries=150, sequenceid=125, filesize=11.7 K 2024-11-09T00:23:38,770 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2edb930c55ccfc4a4303c1efb2c810a9 in 520ms, sequenceid=125, compaction requested=false 2024-11-09T00:23:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-09T00:23:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-09T00:23:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:38,773 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:38,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T00:23:38,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T00:23:38,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-11-09T00:23:38,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2920 sec 2024-11-09T00:23:38,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/4b772f1210c64f27a4bcb8afe4de32e5 is 50, key is test_row_0/A:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742279_1455 (size=14541) 2024-11-09T00:23:38,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/4b772f1210c64f27a4bcb8afe4de32e5 2024-11-09T00:23:38,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/42b94500d84a4ab29250c86ad45cfe3d is 50, key is test_row_0/B:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111878801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111878805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742280_1456 (size=12151) 2024-11-09T00:23:38,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111878914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111878924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111879064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111879064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111879125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111879144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111879194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,201 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/42b94500d84a4ab29250c86ad45cfe3d 2024-11-09T00:23:39,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/75d1058b46cc40789269319f8117f2e7 is 50, key is test_row_0/C:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742281_1457 (size=12151) 2024-11-09T00:23:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111879444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111879454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111879584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111879585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:39,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/75d1058b46cc40789269319f8117f2e7 2024-11-09T00:23:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T00:23:39,634 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:39,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-09T00:23:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T00:23:39,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:39,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:39,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:39,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/4b772f1210c64f27a4bcb8afe4de32e5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5 2024-11-09T00:23:39,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5, entries=200, sequenceid=154, filesize=14.2 K 2024-11-09T00:23:39,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/42b94500d84a4ab29250c86ad45cfe3d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d 2024-11-09T00:23:39,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T00:23:39,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/75d1058b46cc40789269319f8117f2e7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7 2024-11-09T00:23:39,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T00:23:39,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2edb930c55ccfc4a4303c1efb2c810a9 in 883ms, sequenceid=154, compaction requested=true 2024-11-09T00:23:39,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:39,656 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:39,657 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:39,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:39,657 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:39,657 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:39,657 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:39,657 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:39,657 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:39,657 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:39,658 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4d464dec711b413380bd544bc9acd07b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=37.9 K 2024-11-09T00:23:39,658 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff80e35074944389a344b287cd1e4769, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.4 K 2024-11-09T00:23:39,658 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d464dec711b413380bd544bc9acd07b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:39,658 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting ff80e35074944389a344b287cd1e4769, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:39,658 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6714ccd4daea4481882078da2c65ddc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731111816954 2024-11-09T00:23:39,658 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 94fbc9d701f54646be10ff65919fe371, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731111816954 2024-11-09T00:23:39,659 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b94500d84a4ab29250c86ad45cfe3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:39,659 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b772f1210c64f27a4bcb8afe4de32e5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:39,678 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:39,679 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/f54bae01c01b4a7a9d45675da0875251 is 50, key is test_row_0/B:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:39,681 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:39,682 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/cbc0afac93aa486d8b5fe84f493b777d is 50, key is test_row_0/A:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:39,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742282_1458 (size=12391) 2024-11-09T00:23:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742283_1459 (size=12493) 2024-11-09T00:23:39,729 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/cbc0afac93aa486d8b5fe84f493b777d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cbc0afac93aa486d8b5fe84f493b777d 2024-11-09T00:23:39,741 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into cbc0afac93aa486d8b5fe84f493b777d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:39,741 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:39,741 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111819656; duration=0sec 2024-11-09T00:23:39,741 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:39,741 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:39,741 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:39,743 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:39,743 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:39,743 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:39,743 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e220b56d448a4d40bf77d7efaef4d15e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.4 K 2024-11-09T00:23:39,744 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e220b56d448a4d40bf77d7efaef4d15e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731111816246 2024-11-09T00:23:39,744 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c077c325038348938ed1b9c126c10a71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731111816954 2024-11-09T00:23:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T00:23:39,744 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75d1058b46cc40789269319f8117f2e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:39,753 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:39,754 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/75385e8c82bd440dbc90d263a947ef34 is 50, key is test_row_0/C:col10/1731111818772/Put/seqid=0 2024-11-09T00:23:39,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742284_1460 (size=12391) 2024-11-09T00:23:39,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T00:23:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:39,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T00:23:39,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/96cc768d05424907b8cb4cba3f4d4799 is 50, key is test_row_0/A:col10/1731111818791/Put/seqid=0 2024-11-09T00:23:39,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742285_1461 (size=12151) 2024-11-09T00:23:39,836 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/96cc768d05424907b8cb4cba3f4d4799 2024-11-09T00:23:39,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/e0e19e1fd3754fd89b83156353a475f0 is 50, key is test_row_0/B:col10/1731111818791/Put/seqid=0 2024-11-09T00:23:39,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742286_1462 (size=12151) 2024-11-09T00:23:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T00:23:39,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:40,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111880070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111880082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,118 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/f54bae01c01b4a7a9d45675da0875251 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f54bae01c01b4a7a9d45675da0875251 2024-11-09T00:23:40,124 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into f54bae01c01b4a7a9d45675da0875251(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:40,124 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:40,124 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111819657; duration=0sec 2024-11-09T00:23:40,126 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:40,126 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:40,172 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/75385e8c82bd440dbc90d263a947ef34 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75385e8c82bd440dbc90d263a947ef34 2024-11-09T00:23:40,178 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into 75385e8c82bd440dbc90d263a947ef34(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:40,178 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:40,178 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111819657; duration=0sec 2024-11-09T00:23:40,179 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:40,179 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:40,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111880185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111880194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T00:23:40,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/e0e19e1fd3754fd89b83156353a475f0 2024-11-09T00:23:40,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/f5db29ae4a894e899be2b51fa9dc872f is 50, key is test_row_0/C:col10/1731111818791/Put/seqid=0 2024-11-09T00:23:40,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742287_1463 (size=12151) 2024-11-09T00:23:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111880395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111880404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111880605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111880605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,609 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,609 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/f5db29ae4a894e899be2b51fa9dc872f 2024-11-09T00:23:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/96cc768d05424907b8cb4cba3f4d4799 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799 2024-11-09T00:23:40,697 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799, entries=150, sequenceid=165, filesize=11.9 K 2024-11-09T00:23:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/e0e19e1fd3754fd89b83156353a475f0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0 2024-11-09T00:23:40,701 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0, entries=150, sequenceid=165, filesize=11.9 K 2024-11-09T00:23:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/f5db29ae4a894e899be2b51fa9dc872f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f 2024-11-09T00:23:40,706 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-09T00:23:40,707 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 2edb930c55ccfc4a4303c1efb2c810a9 in 916ms, sequenceid=165, compaction requested=false 2024-11-09T00:23:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-09T00:23:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-09T00:23:40,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T00:23:40,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-09T00:23:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:40,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T00:23:40,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0750 sec 2024-11-09T00:23:40,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/ecff6a5d0a8c41e98164a27115c3de3c is 50, key is test_row_0/A:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:40,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742288_1464 (size=14541) 2024-11-09T00:23:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111880731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111880732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T00:23:40,774 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:40,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-09T00:23:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:40,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:40,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:40,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111880844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111880855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:40,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:40,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111881064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111881074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:41,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/ecff6a5d0a8c41e98164a27115c3de3c 2024-11-09T00:23:41,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/634bc848a0e5472ca2b7fe3f5b078ad3 is 50, key is test_row_0/B:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:41,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742289_1465 (size=12151) 2024-11-09T00:23:41,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111881225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,237 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111881384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111881394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:41,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/634bc848a0e5472ca2b7fe3f5b078ad3 2024-11-09T00:23:41,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/8dcf0e1c8ca243f8afaaaf83030bfe91 is 50, key is test_row_0/C:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:41,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742290_1466 (size=12151) 2024-11-09T00:23:41,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:41,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111881905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:41,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111881914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:41,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/8dcf0e1c8ca243f8afaaaf83030bfe91 2024-11-09T00:23:41,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/ecff6a5d0a8c41e98164a27115c3de3c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c 2024-11-09T00:23:41,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c, entries=200, sequenceid=194, filesize=14.2 K 2024-11-09T00:23:41,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/634bc848a0e5472ca2b7fe3f5b078ad3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3 2024-11-09T00:23:41,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3, entries=150, sequenceid=194, filesize=11.9 K 2024-11-09T00:23:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/8dcf0e1c8ca243f8afaaaf83030bfe91 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91 2024-11-09T00:23:41,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91, entries=150, sequenceid=194, filesize=11.9 K 2024-11-09T00:23:41,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1245ms, sequenceid=194, compaction requested=true 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:41,957 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:41,957 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:41,958 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:41,958 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:41,958 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,958 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cbc0afac93aa486d8b5fe84f493b777d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=38.3 K 2024-11-09T00:23:41,959 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:41,959 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:41,959 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,959 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f54bae01c01b4a7a9d45675da0875251, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.8 K 2024-11-09T00:23:41,959 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc0afac93aa486d8b5fe84f493b777d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:41,959 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f54bae01c01b4a7a9d45675da0875251, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:41,959 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96cc768d05424907b8cb4cba3f4d4799, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731111818789 2024-11-09T00:23:41,960 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e19e1fd3754fd89b83156353a475f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731111818789 2024-11-09T00:23:41,960 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 634bc848a0e5472ca2b7fe3f5b078ad3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111820068 2024-11-09T00:23:41,960 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecff6a5d0a8c41e98164a27115c3de3c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111819988 2024-11-09T00:23:41,968 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:41,968 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/76672d798c9a436b8314a186f281560a is 50, key is test_row_0/B:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:41,969 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:41,969 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/aa04632790484398b0bf0d10cbfcd7a6 is 50, key is test_row_0/A:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742291_1467 (size=12493) 2024-11-09T00:23:41,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742292_1468 (size=12595) 2024-11-09T00:23:41,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T00:23:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:41,998 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T00:23:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8732d7d8fca2489ebd3a8f8098b5dc3b is 50, key is test_row_0/A:col10/1731111820731/Put/seqid=0 2024-11-09T00:23:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742293_1469 (size=12151) 2024-11-09T00:23:42,393 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/76672d798c9a436b8314a186f281560a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/76672d798c9a436b8314a186f281560a 2024-11-09T00:23:42,393 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/aa04632790484398b0bf0d10cbfcd7a6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/aa04632790484398b0bf0d10cbfcd7a6 2024-11-09T00:23:42,397 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into aa04632790484398b0bf0d10cbfcd7a6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:42,397 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into 76672d798c9a436b8314a186f281560a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:42,397 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111821957; duration=0sec 2024-11-09T00:23:42,397 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111821957; duration=0sec 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:42,397 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:42,398 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:42,398 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:42,398 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:42,398 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75385e8c82bd440dbc90d263a947ef34, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.8 K 2024-11-09T00:23:42,401 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75385e8c82bd440dbc90d263a947ef34, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731111818404 2024-11-09T00:23:42,401 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5db29ae4a894e899be2b51fa9dc872f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731111818789 2024-11-09T00:23:42,401 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dcf0e1c8ca243f8afaaaf83030bfe91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111820068 2024-11-09T00:23:42,406 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:42,407 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/317019681dbe427498542ca9e7427ca8 is 50, key is test_row_0/C:col10/1731111820710/Put/seqid=0 2024-11-09T00:23:42,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8732d7d8fca2489ebd3a8f8098b5dc3b 2024-11-09T00:23:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/208aa12aba58406091c67b7dc8a6489e is 50, key is test_row_0/B:col10/1731111820731/Put/seqid=0 2024-11-09T00:23:42,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742294_1470 (size=12493) 2024-11-09T00:23:42,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742295_1471 (size=12151) 2024-11-09T00:23:42,430 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/317019681dbe427498542ca9e7427ca8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/317019681dbe427498542ca9e7427ca8 2024-11-09T00:23:42,435 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into 317019681dbe427498542ca9e7427ca8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:42,435 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:42,435 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111821957; duration=0sec 2024-11-09T00:23:42,435 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:42,435 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111882706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:42,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111882710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:42,823 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/208aa12aba58406091c67b7dc8a6489e 2024-11-09T00:23:42,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111882825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/55de899c584547e0baf681e826ce7e51 is 50, key is test_row_0/C:col10/1731111820731/Put/seqid=0 2024-11-09T00:23:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111882825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742296_1472 (size=12151) 2024-11-09T00:23:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:42,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111882925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,927 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:42,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111882934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:42,938 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:42,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111883034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111883034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/55de899c584547e0baf681e826ce7e51 2024-11-09T00:23:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8732d7d8fca2489ebd3a8f8098b5dc3b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b 2024-11-09T00:23:43,239 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T00:23:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/208aa12aba58406091c67b7dc8a6489e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e 2024-11-09T00:23:43,244 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T00:23:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/55de899c584547e0baf681e826ce7e51 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51 2024-11-09T00:23:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51, entries=150, sequenceid=203, filesize=11.9 K 2024-11-09T00:23:43,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1251ms, sequenceid=203, compaction requested=false 2024-11-09T00:23:43,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:43,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:43,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-09T00:23:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-09T00:23:43,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-09T00:23:43,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-11-09T00:23:43,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4780 sec 2024-11-09T00:23:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:43,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:43,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T00:23:43,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8696a47fe3ef48fd945338912450fa15 is 50, key is test_row_0/A:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742297_1473 (size=16931) 2024-11-09T00:23:43,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111883368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111883373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111883484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111883484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111883694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111883694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:43,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8696a47fe3ef48fd945338912450fa15 2024-11-09T00:23:43,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/375dde6872d844f3805cf91c28aa6bc4 is 50, key is test_row_0/B:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742298_1474 (size=12151) 2024-11-09T00:23:43,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/375dde6872d844f3805cf91c28aa6bc4 2024-11-09T00:23:43,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/73d6b881096a42cda4052cf3710741b7 is 50, key is test_row_0/C:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742299_1475 (size=12151) 2024-11-09T00:23:43,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/73d6b881096a42cda4052cf3710741b7 2024-11-09T00:23:43,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/8696a47fe3ef48fd945338912450fa15 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15 2024-11-09T00:23:43,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15, entries=250, sequenceid=235, filesize=16.5 K 2024-11-09T00:23:43,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/375dde6872d844f3805cf91c28aa6bc4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4 2024-11-09T00:23:43,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-09T00:23:43,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/73d6b881096a42cda4052cf3710741b7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7 2024-11-09T00:23:43,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7, entries=150, sequenceid=235, filesize=11.9 K 2024-11-09T00:23:43,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 2edb930c55ccfc4a4303c1efb2c810a9 in 487ms, sequenceid=235, compaction requested=true 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:43,838 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:43,838 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:43,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:43,839 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:43,839 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:43,839 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:43,839 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/aa04632790484398b0bf0d10cbfcd7a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=40.7 K 2024-11-09T00:23:43,840 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:43,840 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:43,840 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:43,840 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/76672d798c9a436b8314a186f281560a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.9 K 2024-11-09T00:23:43,841 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 76672d798c9a436b8314a186f281560a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111820068 2024-11-09T00:23:43,842 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa04632790484398b0bf0d10cbfcd7a6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111820068 2024-11-09T00:23:43,842 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8732d7d8fca2489ebd3a8f8098b5dc3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731111820727 2024-11-09T00:23:43,843 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 208aa12aba58406091c67b7dc8a6489e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731111820727 2024-11-09T00:23:43,843 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8696a47fe3ef48fd945338912450fa15, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822704 2024-11-09T00:23:43,843 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 375dde6872d844f3805cf91c28aa6bc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822705 2024-11-09T00:23:43,860 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#401 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:43,861 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/b952fe623c754d9a88f34b753d8eb433 is 50, key is test_row_0/A:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,861 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:43,862 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/37b94d7d37ba4bd6bf5ce218040c6a75 is 50, key is test_row_0/B:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742300_1476 (size=12697) 2024-11-09T00:23:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742301_1477 (size=12595) 2024-11-09T00:23:43,900 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/b952fe623c754d9a88f34b753d8eb433 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b952fe623c754d9a88f34b753d8eb433 2024-11-09T00:23:43,905 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/37b94d7d37ba4bd6bf5ce218040c6a75 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/37b94d7d37ba4bd6bf5ce218040c6a75 2024-11-09T00:23:43,910 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into b952fe623c754d9a88f34b753d8eb433(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:43,910 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:43,910 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111823838; duration=0sec 2024-11-09T00:23:43,910 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:43,910 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:43,910 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:43,912 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:43,912 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:43,912 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:43,912 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/317019681dbe427498542ca9e7427ca8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=35.9 K 2024-11-09T00:23:43,913 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317019681dbe427498542ca9e7427ca8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111820068 2024-11-09T00:23:43,913 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55de899c584547e0baf681e826ce7e51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731111820727 2024-11-09T00:23:43,915 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into 37b94d7d37ba4bd6bf5ce218040c6a75(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:43,915 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:43,915 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111823838; duration=0sec 2024-11-09T00:23:43,915 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:43,915 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:43,916 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d6b881096a42cda4052cf3710741b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822705 2024-11-09T00:23:43,928 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:43,928 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2da82675053246fc9c43c8bc4b2f66dc is 50, key is test_row_0/C:col10/1731111822705/Put/seqid=0 2024-11-09T00:23:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742302_1478 (size=12595) 2024-11-09T00:23:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:44,011 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:44,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:44,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c54b787fbc744b0eb9e749d4a0c3e33e is 50, key is test_row_0/A:col10/1731111823367/Put/seqid=0 2024-11-09T00:23:44,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742303_1479 (size=16931) 2024-11-09T00:23:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111884097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111884104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111884214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111884225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,339 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2da82675053246fc9c43c8bc4b2f66dc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2da82675053246fc9c43c8bc4b2f66dc 2024-11-09T00:23:44,344 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into 2da82675053246fc9c43c8bc4b2f66dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:44,344 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:44,344 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111823838; duration=0sec 2024-11-09T00:23:44,344 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:44,344 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:44,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c54b787fbc744b0eb9e749d4a0c3e33e 2024-11-09T00:23:44,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/1e05c6bbbd19475eaff97bdfc57d8adc is 50, key is test_row_0/B:col10/1731111823367/Put/seqid=0 2024-11-09T00:23:44,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742304_1480 (size=12151) 2024-11-09T00:23:44,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111884425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111884434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111884744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111884754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/1e05c6bbbd19475eaff97bdfc57d8adc 2024-11-09T00:23:44,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/6e6147503faf4fe79ea154fcb93775dd is 50, key is test_row_0/C:col10/1731111823367/Put/seqid=0 2024-11-09T00:23:44,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742305_1481 (size=12151) 2024-11-09T00:23:44,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/6e6147503faf4fe79ea154fcb93775dd 2024-11-09T00:23:44,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c54b787fbc744b0eb9e749d4a0c3e33e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e 2024-11-09T00:23:44,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e, entries=250, sequenceid=248, filesize=16.5 K 2024-11-09T00:23:44,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/1e05c6bbbd19475eaff97bdfc57d8adc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc 2024-11-09T00:23:44,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc, entries=150, sequenceid=248, filesize=11.9 K 2024-11-09T00:23:44,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/6e6147503faf4fe79ea154fcb93775dd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd 2024-11-09T00:23:44,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd, entries=150, sequenceid=248, filesize=11.9 K 2024-11-09T00:23:44,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2edb930c55ccfc4a4303c1efb2c810a9 in 851ms, sequenceid=248, compaction requested=false 2024-11-09T00:23:44,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T00:23:44,934 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:44,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-09T00:23:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:44,938 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:44,938 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:44,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:44,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T00:23:44,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5f504aa6d46648419cfd764b6d741dce is 50, key is test_row_0/A:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:44,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742306_1482 (size=14741) 2024-11-09T00:23:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111884976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111884977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:45,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111885095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111885095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111885254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34148 deadline: 1731111885264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111885264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,272 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111885314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111885314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5f504aa6d46648419cfd764b6d741dce 2024-11-09T00:23:45,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/281f4b2b9fd84187b3308aeb1bcdaa7e is 50, key is test_row_0/B:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:45,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742307_1483 (size=12301) 2024-11-09T00:23:45,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111885635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111885635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:45,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/281f4b2b9fd84187b3308aeb1bcdaa7e 2024-11-09T00:23:45,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/5f6acce87fe74b8187a1d12eacc31cf0 is 50, key is test_row_0/C:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:45,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742308_1484 (size=12301) 2024-11-09T00:23:45,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:45,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111886144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111886155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/5f6acce87fe74b8187a1d12eacc31cf0 2024-11-09T00:23:46,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5f504aa6d46648419cfd764b6d741dce as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce 2024-11-09T00:23:46,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce, entries=200, sequenceid=275, filesize=14.4 K 2024-11-09T00:23:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/281f4b2b9fd84187b3308aeb1bcdaa7e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e 2024-11-09T00:23:46,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e, entries=150, sequenceid=275, filesize=12.0 K 2024-11-09T00:23:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/5f6acce87fe74b8187a1d12eacc31cf0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0 2024-11-09T00:23:46,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-09T00:23:46,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1245ms, sequenceid=275, compaction requested=true 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:46,194 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:46,194 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:46,195 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:46,195 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:46,195 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,195 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b952fe623c754d9a88f34b753d8eb433, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=43.3 K 2024-11-09T00:23:46,196 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:46,197 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:46,197 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,197 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/37b94d7d37ba4bd6bf5ce218040c6a75, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.2 K 2024-11-09T00:23:46,198 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b952fe623c754d9a88f34b753d8eb433, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822705 2024-11-09T00:23:46,198 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b94d7d37ba4bd6bf5ce218040c6a75, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822705 2024-11-09T00:23:46,198 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c54b787fbc744b0eb9e749d4a0c3e33e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731111823367 2024-11-09T00:23:46,198 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e05c6bbbd19475eaff97bdfc57d8adc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731111823367 2024-11-09T00:23:46,198 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f504aa6d46648419cfd764b6d741dce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824068 2024-11-09T00:23:46,199 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 281f4b2b9fd84187b3308aeb1bcdaa7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824090 2024-11-09T00:23:46,218 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:46,219 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/649c259abab946bd974d01f4ea9e4ce8 is 50, key is test_row_0/A:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:46,226 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:46,227 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a03044e7aba441c19b95a9a2e8621330 is 50, key is test_row_0/B:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:46,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742309_1485 (size=12949) 2024-11-09T00:23:46,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742310_1486 (size=12847) 2024-11-09T00:23:46,245 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a03044e7aba441c19b95a9a2e8621330 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a03044e7aba441c19b95a9a2e8621330 2024-11-09T00:23:46,251 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into a03044e7aba441c19b95a9a2e8621330(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:46,251 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:46,251 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111826194; duration=0sec 2024-11-09T00:23:46,251 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:46,251 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:46,251 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:46,253 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:46,253 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:46,253 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,253 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2da82675053246fc9c43c8bc4b2f66dc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.2 K 2024-11-09T00:23:46,254 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da82675053246fc9c43c8bc4b2f66dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731111822705 2024-11-09T00:23:46,254 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6147503faf4fe79ea154fcb93775dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731111823367 2024-11-09T00:23:46,254 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6acce87fe74b8187a1d12eacc31cf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824090 2024-11-09T00:23:46,261 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:46,261 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/a310cacfe30c4b748790334ee66163d1 is 50, key is test_row_0/C:col10/1731111824949/Put/seqid=0 2024-11-09T00:23:46,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742311_1487 (size=12847) 2024-11-09T00:23:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:46,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:46,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:23:46,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/858b45c5cca144b7b1a080ce36cc7475 is 50, key is test_row_0/A:col10/1731111826278/Put/seqid=0 2024-11-09T00:23:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742312_1488 (size=14741) 2024-11-09T00:23:46,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/858b45c5cca144b7b1a080ce36cc7475 2024-11-09T00:23:46,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/74680bd6d50a427bbe1ad24c8f9a210e is 50, key is test_row_0/B:col10/1731111826278/Put/seqid=0 2024-11-09T00:23:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742313_1489 (size=12301) 2024-11-09T00:23:46,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/74680bd6d50a427bbe1ad24c8f9a210e 2024-11-09T00:23:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/427768a801c54fa49dd0949d65c45594 is 50, key is test_row_0/C:col10/1731111826278/Put/seqid=0 2024-11-09T00:23:46,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742314_1490 (size=12301) 2024-11-09T00:23:46,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/427768a801c54fa49dd0949d65c45594 2024-11-09T00:23:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/858b45c5cca144b7b1a080ce36cc7475 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475 2024-11-09T00:23:46,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475, entries=200, sequenceid=287, filesize=14.4 K 2024-11-09T00:23:46,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/74680bd6d50a427bbe1ad24c8f9a210e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e 2024-11-09T00:23:46,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e, entries=150, sequenceid=287, filesize=12.0 K 2024-11-09T00:23:46,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/427768a801c54fa49dd0949d65c45594 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594 2024-11-09T00:23:46,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594, entries=150, sequenceid=287, filesize=12.0 K 2024-11-09T00:23:46,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=87.22 KB/89310 for 2edb930c55ccfc4a4303c1efb2c810a9 in 70ms, sequenceid=287, compaction requested=false 2024-11-09T00:23:46,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:46,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T00:23:46,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/d3ea174446da4f63a7f352207994afca is 50, key is test_row_0/A:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742315_1491 (size=14741) 2024-11-09T00:23:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111886431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111886432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111886535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111886554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,641 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/649c259abab946bd974d01f4ea9e4ce8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/649c259abab946bd974d01f4ea9e4ce8 2024-11-09T00:23:46,645 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into 649c259abab946bd974d01f4ea9e4ce8(size=12.6 K), total size for store is 27.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:46,645 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:46,645 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111826194; duration=0sec 2024-11-09T00:23:46,645 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:46,646 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:46,673 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/a310cacfe30c4b748790334ee66163d1 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/a310cacfe30c4b748790334ee66163d1 2024-11-09T00:23:46,677 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into a310cacfe30c4b748790334ee66163d1(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:46,677 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:46,677 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111826194; duration=0sec 2024-11-09T00:23:46,677 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:46,677 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:46,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111886744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/d3ea174446da4f63a7f352207994afca 2024-11-09T00:23:46,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/36c0dd40fe1a42d385ef1052e6c490da is 50, key is test_row_0/B:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:46,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742316_1492 (size=12301) 2024-11-09T00:23:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111886765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:46,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:46,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:47,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111887054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111887084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111887164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/36c0dd40fe1a42d385ef1052e6c490da 2024-11-09T00:23:47,173 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/68f3812b1dce4b18827387d8a0a436de is 50, key is test_row_0/C:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742317_1493 (size=12301) 2024-11-09T00:23:47,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/68f3812b1dce4b18827387d8a0a436de 2024-11-09T00:23:47,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111887174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,184 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/d3ea174446da4f63a7f352207994afca as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca 2024-11-09T00:23:47,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca, entries=200, sequenceid=304, filesize=14.4 K 2024-11-09T00:23:47,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/36c0dd40fe1a42d385ef1052e6c490da as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da 2024-11-09T00:23:47,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da, entries=150, sequenceid=304, filesize=12.0 K 2024-11-09T00:23:47,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/68f3812b1dce4b18827387d8a0a436de as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de 2024-11-09T00:23:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de, entries=150, sequenceid=304, filesize=12.0 K 2024-11-09T00:23:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2edb930c55ccfc4a4303c1efb2c810a9 in 843ms, sequenceid=304, compaction requested=true 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:47,199 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:47,199 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:47,200 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,200 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,200 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a03044e7aba441c19b95a9a2e8621330, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.6 K 2024-11-09T00:23:47,200 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/649c259abab946bd974d01f4ea9e4ce8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=41.4 K 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649c259abab946bd974d01f4ea9e4ce8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824090 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a03044e7aba441c19b95a9a2e8621330, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824090 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 74680bd6d50a427bbe1ad24c8f9a210e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731111824963 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858b45c5cca144b7b1a080ce36cc7475, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731111824963 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 36c0dd40fe1a42d385ef1052e6c490da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:47,200 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ea174446da4f63a7f352207994afca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:47,206 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:47,207 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/3fa15f955cd048eaa33810e3d292fb7c is 50, key is test_row_0/A:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:47,214 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:47,215 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/e09d57d2ccad4e77a33aefd547728a6b is 50, key is test_row_0/B:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742319_1495 (size=12949) 2024-11-09T00:23:47,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742318_1494 (size=13051) 2024-11-09T00:23:47,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T00:23:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T00:23:47,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/112f5c889b254172b268d2674bb94cfa is 50, key is test_row_0/A:col10/1731111826420/Put/seqid=0 2024-11-09T00:23:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742320_1496 (size=12301) 2024-11-09T00:23:47,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:47,626 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/e09d57d2ccad4e77a33aefd547728a6b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e09d57d2ccad4e77a33aefd547728a6b 2024-11-09T00:23:47,626 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/3fa15f955cd048eaa33810e3d292fb7c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/3fa15f955cd048eaa33810e3d292fb7c 2024-11-09T00:23:47,630 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into e09d57d2ccad4e77a33aefd547728a6b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:47,630 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into 3fa15f955cd048eaa33810e3d292fb7c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:47,630 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111827199; duration=0sec 2024-11-09T00:23:47,630 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111827199; duration=0sec 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:47,630 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:47,631 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:47,631 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:47,631 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:47,631 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/a310cacfe30c4b748790334ee66163d1, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.6 K 2024-11-09T00:23:47,631 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a310cacfe30c4b748790334ee66163d1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731111824090 2024-11-09T00:23:47,632 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 427768a801c54fa49dd0949d65c45594, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731111824963 2024-11-09T00:23:47,632 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f3812b1dce4b18827387d8a0a436de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:47,636 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:47,637 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2d259c0d2e3c462386e079af2ec976b0 is 50, key is test_row_0/C:col10/1731111826337/Put/seqid=0 2024-11-09T00:23:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742321_1497 (size=12949) 2024-11-09T00:23:47,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/112f5c889b254172b268d2674bb94cfa 2024-11-09T00:23:47,645 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/2d259c0d2e3c462386e079af2ec976b0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2d259c0d2e3c462386e079af2ec976b0 2024-11-09T00:23:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a98951d3ddf44357a5451dbeb9b87ef4 is 50, key is test_row_0/B:col10/1731111826420/Put/seqid=0 2024-11-09T00:23:47,653 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into 2d259c0d2e3c462386e079af2ec976b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:47,653 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:47,653 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111827199; duration=0sec 2024-11-09T00:23:47,653 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:47,653 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:47,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742322_1498 (size=12301) 2024-11-09T00:23:47,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111887641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111887643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111887764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111887764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111887974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:47,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111887974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:48,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a98951d3ddf44357a5451dbeb9b87ef4 2024-11-09T00:23:48,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d1ac40e9d31b4f8a9c7dc2884b024545 is 50, key is test_row_0/C:col10/1731111826420/Put/seqid=0 2024-11-09T00:23:48,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742323_1499 (size=12301) 2024-11-09T00:23:48,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111888294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:48,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111888294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:48,464 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d1ac40e9d31b4f8a9c7dc2884b024545 2024-11-09T00:23:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/112f5c889b254172b268d2674bb94cfa as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa 2024-11-09T00:23:48,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T00:23:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/a98951d3ddf44357a5451dbeb9b87ef4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4 2024-11-09T00:23:48,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T00:23:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d1ac40e9d31b4f8a9c7dc2884b024545 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545 2024-11-09T00:23:48,478 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T00:23:48,479 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1245ms, sequenceid=325, compaction requested=false 2024-11-09T00:23:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-09T00:23:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-09T00:23:48,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T00:23:48,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5420 sec 2024-11-09T00:23:48,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 3.5470 sec 2024-11-09T00:23:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:48,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:48,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T00:23:48,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c421e07d9c714a4b934fc2e24e39b784 is 50, key is test_row_0/A:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:48,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742324_1500 (size=14741) 2024-11-09T00:23:48,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c421e07d9c714a4b934fc2e24e39b784 2024-11-09T00:23:48,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/d5f4ef7bcced402fbd54e0ee1b217477 is 50, key is test_row_0/B:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:48,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742325_1501 (size=12301) 2024-11-09T00:23:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111888883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111888899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111889004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111889014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T00:23:49,095 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:49,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:23:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T00:23:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:49,097 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:23:49,097 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:23:49,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:23:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34082 deadline: 1731111889194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34132 deadline: 1731111889195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,203 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_2' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:49,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,203 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in 2edb930c55ccfc4a4303c1efb2c810a9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:23:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111889224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111889234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/d5f4ef7bcced402fbd54e0ee1b217477 2024-11-09T00:23:49,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T00:23:49,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:49,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/dcb2aadc155a4e75ae499e887129e836 is 50, key is test_row_0/C:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742326_1502 (size=12301) 2024-11-09T00:23:49,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T00:23:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:49,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111889545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T00:23:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:23:49,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111889554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:49,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/dcb2aadc155a4e75ae499e887129e836 2024-11-09T00:23:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c421e07d9c714a4b934fc2e24e39b784 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784 2024-11-09T00:23:49,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784, entries=200, sequenceid=346, filesize=14.4 K 2024-11-09T00:23:49,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/d5f4ef7bcced402fbd54e0ee1b217477 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477 2024-11-09T00:23:49,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477, entries=150, sequenceid=346, filesize=12.0 K 2024-11-09T00:23:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/dcb2aadc155a4e75ae499e887129e836 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836 2024-11-09T00:23:49,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836, entries=150, sequenceid=346, filesize=12.0 K 2024-11-09T00:23:49,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2edb930c55ccfc4a4303c1efb2c810a9 in 875ms, sequenceid=346, compaction requested=true 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:49,697 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:49,697 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:49,702 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:49,702 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:49,702 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,702 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:49,702 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:49,702 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e09d57d2ccad4e77a33aefd547728a6b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.7 K 2024-11-09T00:23:49,702 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,702 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/3fa15f955cd048eaa33810e3d292fb7c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=39.2 K 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting e09d57d2ccad4e77a33aefd547728a6b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fa15f955cd048eaa33810e3d292fb7c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a98951d3ddf44357a5451dbeb9b87ef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731111826420 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 112f5c889b254172b268d2674bb94cfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731111826420 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f4ef7bcced402fbd54e0ee1b217477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827630 2024-11-09T00:23:49,703 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c421e07d9c714a4b934fc2e24e39b784, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827623 2024-11-09T00:23:49,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T00:23:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,711 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:49,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T00:23:49,711 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:49,711 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/1d02aa58d1ac4794b527c4717eb1ee30 is 50, key is test_row_0/B:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:49,711 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/e86613d577754f02a3f5813d64bfdaf0 is 50, key is test_row_0/A:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/515f5811552847af80b2fea90a84a363 is 50, key is test_row_0/A:col10/1731111828874/Put/seqid=0 2024-11-09T00:23:49,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742327_1503 (size=13153) 2024-11-09T00:23:49,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742328_1504 (size=12301) 2024-11-09T00:23:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742329_1505 (size=13051) 2024-11-09T00:23:49,735 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/1d02aa58d1ac4794b527c4717eb1ee30 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1d02aa58d1ac4794b527c4717eb1ee30 2024-11-09T00:23:49,739 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into 1d02aa58d1ac4794b527c4717eb1ee30(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:49,739 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:49,739 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111829697; duration=0sec 2024-11-09T00:23:49,739 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:49,739 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:49,739 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:49,740 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:49,740 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:49,740 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:49,740 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2d259c0d2e3c462386e079af2ec976b0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.7 K 2024-11-09T00:23:49,740 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d259c0d2e3c462386e079af2ec976b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731111826321 2024-11-09T00:23:49,740 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ac40e9d31b4f8a9c7dc2884b024545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731111826420 2024-11-09T00:23:49,741 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb2aadc155a4e75ae499e887129e836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827630 2024-11-09T00:23:49,746 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:49,746 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/ffc21e15319d4304a0fb6347e0a57ddb is 50, key is test_row_0/C:col10/1731111828819/Put/seqid=0 2024-11-09T00:23:49,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742330_1506 (size=13051) 2024-11-09T00:23:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:50,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. as already flushing 2024-11-09T00:23:50,122 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/515f5811552847af80b2fea90a84a363 2024-11-09T00:23:50,125 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/e86613d577754f02a3f5813d64bfdaf0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/e86613d577754f02a3f5813d64bfdaf0 2024-11-09T00:23:50,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/28750ec5c54543e991c24136008c63be is 50, key is test_row_0/B:col10/1731111828874/Put/seqid=0 2024-11-09T00:23:50,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742331_1507 (size=12301) 2024-11-09T00:23:50,132 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into e86613d577754f02a3f5813d64bfdaf0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:50,132 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:50,132 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111829697; duration=0sec 2024-11-09T00:23:50,132 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:50,132 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:50,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111890135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111890136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,170 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/ffc21e15319d4304a0fb6347e0a57ddb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ffc21e15319d4304a0fb6347e0a57ddb 2024-11-09T00:23:50,176 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into ffc21e15319d4304a0fb6347e0a57ddb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:50,176 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:50,176 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111829697; duration=0sec 2024-11-09T00:23:50,177 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:50,177 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111890254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111890255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111890464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111890464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,532 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/28750ec5c54543e991c24136008c63be 2024-11-09T00:23:50,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d632f7c0092d4bec8d2af54b55c565e7 is 50, key is test_row_0/C:col10/1731111828874/Put/seqid=0 2024-11-09T00:23:50,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742332_1508 (size=12301) 2024-11-09T00:23:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34098 deadline: 1731111890775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34054 deadline: 1731111890775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 , the old value is region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:23:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., hostname=ef986b38239e,35861,1731111692607, seqNum=2 because the exception is null or not the one we care about 2024-11-09T00:23:50,943 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d632f7c0092d4bec8d2af54b55c565e7 2024-11-09T00:23:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/515f5811552847af80b2fea90a84a363 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363 2024-11-09T00:23:50,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363, entries=150, sequenceid=364, filesize=12.0 K 2024-11-09T00:23:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/28750ec5c54543e991c24136008c63be as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be 2024-11-09T00:23:50,961 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be, entries=150, sequenceid=364, filesize=12.0 K 2024-11-09T00:23:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d632f7c0092d4bec8d2af54b55c565e7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7 2024-11-09T00:23:50,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7, entries=150, sequenceid=364, filesize=12.0 K 2024-11-09T00:23:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1254ms, sequenceid=364, compaction requested=false 2024-11-09T00:23:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T00:23:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T00:23:50,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T00:23:50,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-09T00:23:50,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8740 sec 2024-11-09T00:23:51,175 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-09T00:23:51,175 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,175 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,175 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,177 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-09T00:23:51,177 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,177 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,177 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,178 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-09T00:23:51,178 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,178 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,178 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,182 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-09T00:23:51,182 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,182 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,182 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,183 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-09T00:23:51,183 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,183 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,183 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T00:23:51,245 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:23:51,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T00:23:51,286 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-09T00:23:51,286 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,286 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,286 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5c1967e272b645e3b95831fc1f3503ae is 50, key is test_row_0/A:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742333_1509 (size=12301) 2024-11-09T00:23:51,295 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-11-09T00:23:51,295 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:51,295 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,296 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:51,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:51,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5c1967e272b645e3b95831fc1f3503ae 2024-11-09T00:23:51,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/33a5bfdf624942f7b9c256355680c7c5 is 50, key is test_row_0/B:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:51,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742334_1510 (size=12301) 2024-11-09T00:23:52,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/33a5bfdf624942f7b9c256355680c7c5 2024-11-09T00:23:52,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/e28b34762dac488ca4c4b1e4cac6d22d is 50, key is test_row_0/C:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742335_1511 (size=12301) 2024-11-09T00:23:52,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/e28b34762dac488ca4c4b1e4cac6d22d 2024-11-09T00:23:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/5c1967e272b645e3b95831fc1f3503ae as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae 2024-11-09T00:23:52,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae, entries=150, sequenceid=386, filesize=12.0 K 2024-11-09T00:23:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/33a5bfdf624942f7b9c256355680c7c5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5 2024-11-09T00:23:52,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5, entries=150, sequenceid=386, filesize=12.0 K 2024-11-09T00:23:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/e28b34762dac488ca4c4b1e4cac6d22d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d 2024-11-09T00:23:52,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d, entries=150, sequenceid=386, filesize=12.0 K 2024-11-09T00:23:52,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1262ms, sequenceid=386, compaction requested=true 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:23:52,549 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2edb930c55ccfc4a4303c1efb2c810a9:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:23:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:52,549 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/B is initiating minor compaction (all files) 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/A is initiating minor compaction (all files) 2024-11-09T00:23:52,550 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/B in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:52,550 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/A in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:52,550 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1d02aa58d1ac4794b527c4717eb1ee30, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.8 K 2024-11-09T00:23:52,550 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/e86613d577754f02a3f5813d64bfdaf0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.9 K 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d02aa58d1ac4794b527c4717eb1ee30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827630 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86613d577754f02a3f5813d64bfdaf0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827630 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 28750ec5c54543e991c24136008c63be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731111828874 2024-11-09T00:23:52,550 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 515f5811552847af80b2fea90a84a363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731111828874 2024-11-09T00:23:52,551 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a5bfdf624942f7b9c256355680c7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731111830131 2024-11-09T00:23:52,551 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1967e272b645e3b95831fc1f3503ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731111830131 2024-11-09T00:23:52,558 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:52,558 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:52,558 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/3aa6e0a28e81422b85028fc94436d0f0 is 50, key is test_row_0/B:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:52,558 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c11c1d40b1904f0a9b55e52a26461660 is 50, key is test_row_0/A:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:52,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742336_1512 (size=13255) 2024-11-09T00:23:52,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742337_1513 (size=13153) 2024-11-09T00:23:52,575 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/c11c1d40b1904f0a9b55e52a26461660 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c11c1d40b1904f0a9b55e52a26461660 2024-11-09T00:23:52,575 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/3aa6e0a28e81422b85028fc94436d0f0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/3aa6e0a28e81422b85028fc94436d0f0 2024-11-09T00:23:52,578 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/A of 2edb930c55ccfc4a4303c1efb2c810a9 into c11c1d40b1904f0a9b55e52a26461660(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:52,578 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/B of 2edb930c55ccfc4a4303c1efb2c810a9 into 3aa6e0a28e81422b85028fc94436d0f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:52,579 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/A, priority=13, startTime=1731111832549; duration=0sec 2024-11-09T00:23:52,579 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/B, priority=13, startTime=1731111832549; duration=0sec 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:A 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:B 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:23:52,579 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): 2edb930c55ccfc4a4303c1efb2c810a9/C is initiating minor compaction (all files) 2024-11-09T00:23:52,579 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2edb930c55ccfc4a4303c1efb2c810a9/C in TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:52,579 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ffc21e15319d4304a0fb6347e0a57ddb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp, totalSize=36.8 K 2024-11-09T00:23:52,580 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc21e15319d4304a0fb6347e0a57ddb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731111827630 2024-11-09T00:23:52,580 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d632f7c0092d4bec8d2af54b55c565e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731111828874 2024-11-09T00:23:52,580 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28b34762dac488ca4c4b1e4cac6d22d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731111830131 2024-11-09T00:23:52,585 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2edb930c55ccfc4a4303c1efb2c810a9#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:23:52,585 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/65839deca69141d49e01866a2839688a is 50, key is test_row_0/C:col10/1731111830131/Put/seqid=0 2024-11-09T00:23:52,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742338_1514 (size=13153) 2024-11-09T00:23:52,996 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/65839deca69141d49e01866a2839688a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/65839deca69141d49e01866a2839688a 2024-11-09T00:23:53,001 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2edb930c55ccfc4a4303c1efb2c810a9/C of 2edb930c55ccfc4a4303c1efb2c810a9 into 65839deca69141d49e01866a2839688a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:23:53,001 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2edb930c55ccfc4a4303c1efb2c810a9: 2024-11-09T00:23:53,001 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., storeName=2edb930c55ccfc4a4303c1efb2c810a9/C, priority=13, startTime=1731111832549; duration=0sec 2024-11-09T00:23:53,001 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:23:53,002 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2edb930c55ccfc4a4303c1efb2c810a9:C 2024-11-09T00:23:53,238 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-09T00:23:53,238 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-11-09T00:23:53,238 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:53,238 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:53,238 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:53,239 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:53,239 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:53,239 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:53,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:53,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:55,318 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-11-09T00:23:55,318 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:23:55,318 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:55,318 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:55,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1243 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3720 rows 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1235 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3704 rows 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1241 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3717 rows 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1235 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3703 rows 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1243 2024-11-09T00:23:55,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3728 rows 2024-11-09T00:23:55,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:23:55,319 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:23:55,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:55,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:55,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:23:55,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:23:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:55,325 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111835325"}]},"ts":"1731111835325"} 2024-11-09T00:23:55,327 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:23:55,327 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:23:55,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:23:55,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, UNASSIGN}] 2024-11-09T00:23:55,333 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, UNASSIGN 2024-11-09T00:23:55,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2edb930c55ccfc4a4303c1efb2c810a9, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:55,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, UNASSIGN because future has completed 2024-11-09T00:23:55,335 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:23:55,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:55,488 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:55,488 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:23:55,488 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 2edb930c55ccfc4a4303c1efb2c810a9, disabling compactions & flushes 2024-11-09T00:23:55,488 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:55,488 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:55,488 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. after waiting 0 ms 2024-11-09T00:23:55,488 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:55,488 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 2edb930c55ccfc4a4303c1efb2c810a9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T00:23:55,492 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/f95797df8fbb444981cf1a6c9f20ebff is 50, key is test_row_1/A:col10/1731111835316/Put/seqid=0 2024-11-09T00:23:55,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742339_1515 (size=9857) 2024-11-09T00:23:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:55,899 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/f95797df8fbb444981cf1a6c9f20ebff 2024-11-09T00:23:55,909 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/f7bdccaf51e04445829721c8075a7428 is 50, key is test_row_1/B:col10/1731111835316/Put/seqid=0 2024-11-09T00:23:55,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742340_1516 (size=9857) 2024-11-09T00:23:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:56,316 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/f7bdccaf51e04445829721c8075a7428 2024-11-09T00:23:56,329 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d00430c0195a4559b8aed84e276cef88 is 50, key is test_row_1/C:col10/1731111835316/Put/seqid=0 2024-11-09T00:23:56,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742341_1517 (size=9857) 2024-11-09T00:23:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:56,736 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d00430c0195a4559b8aed84e276cef88 2024-11-09T00:23:56,743 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/A/f95797df8fbb444981cf1a6c9f20ebff as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/f95797df8fbb444981cf1a6c9f20ebff 2024-11-09T00:23:56,747 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/f95797df8fbb444981cf1a6c9f20ebff, entries=100, sequenceid=396, filesize=9.6 K 2024-11-09T00:23:56,747 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/B/f7bdccaf51e04445829721c8075a7428 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f7bdccaf51e04445829721c8075a7428 2024-11-09T00:23:56,750 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f7bdccaf51e04445829721c8075a7428, entries=100, sequenceid=396, filesize=9.6 K 2024-11-09T00:23:56,751 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/.tmp/C/d00430c0195a4559b8aed84e276cef88 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d00430c0195a4559b8aed84e276cef88 2024-11-09T00:23:56,754 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d00430c0195a4559b8aed84e276cef88, entries=100, sequenceid=396, filesize=9.6 K 2024-11-09T00:23:56,755 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1267ms, sequenceid=396, compaction requested=false 2024-11-09T00:23:56,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cdb077836c424030864a00b78eb786f2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4d464dec711b413380bd544bc9acd07b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cbc0afac93aa486d8b5fe84f493b777d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/aa04632790484398b0bf0d10cbfcd7a6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b952fe623c754d9a88f34b753d8eb433, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/649c259abab946bd974d01f4ea9e4ce8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/3fa15f955cd048eaa33810e3d292fb7c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/e86613d577754f02a3f5813d64bfdaf0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae] to archive 2024-11-09T00:23:56,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:56,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b8d28970a3bf4fbf9771b37ba23bb68b 2024-11-09T00:23:56,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/7118fec1a24e4d71a675cf3de185ab91 2024-11-09T00:23:56,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8d48b483332044af989588b173484467 2024-11-09T00:23:56,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cdb077836c424030864a00b78eb786f2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cdb077836c424030864a00b78eb786f2 2024-11-09T00:23:56,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/a554c7c5a71344f9ad150009b3893f03 2024-11-09T00:23:56,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/537e9c258d9948489c3daffea67fd011 2024-11-09T00:23:56,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/42cc8ddee7c5410989d9cc9fac507cf4 2024-11-09T00:23:56,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4d464dec711b413380bd544bc9acd07b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4d464dec711b413380bd544bc9acd07b 2024-11-09T00:23:56,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/6714ccd4daea4481882078da2c65ddc9 2024-11-09T00:23:56,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/4b772f1210c64f27a4bcb8afe4de32e5 2024-11-09T00:23:56,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cbc0afac93aa486d8b5fe84f493b777d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/cbc0afac93aa486d8b5fe84f493b777d 2024-11-09T00:23:56,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/96cc768d05424907b8cb4cba3f4d4799 2024-11-09T00:23:56,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/ecff6a5d0a8c41e98164a27115c3de3c 2024-11-09T00:23:56,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/aa04632790484398b0bf0d10cbfcd7a6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/aa04632790484398b0bf0d10cbfcd7a6 2024-11-09T00:23:56,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8732d7d8fca2489ebd3a8f8098b5dc3b 2024-11-09T00:23:56,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/8696a47fe3ef48fd945338912450fa15 2024-11-09T00:23:56,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b952fe623c754d9a88f34b753d8eb433 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/b952fe623c754d9a88f34b753d8eb433 2024-11-09T00:23:56,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c54b787fbc744b0eb9e749d4a0c3e33e 2024-11-09T00:23:56,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5f504aa6d46648419cfd764b6d741dce 2024-11-09T00:23:56,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/649c259abab946bd974d01f4ea9e4ce8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/649c259abab946bd974d01f4ea9e4ce8 2024-11-09T00:23:56,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/858b45c5cca144b7b1a080ce36cc7475 2024-11-09T00:23:56,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/d3ea174446da4f63a7f352207994afca 2024-11-09T00:23:56,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/3fa15f955cd048eaa33810e3d292fb7c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/3fa15f955cd048eaa33810e3d292fb7c 2024-11-09T00:23:56,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/112f5c889b254172b268d2674bb94cfa 2024-11-09T00:23:56,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c421e07d9c714a4b934fc2e24e39b784 2024-11-09T00:23:56,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/e86613d577754f02a3f5813d64bfdaf0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/e86613d577754f02a3f5813d64bfdaf0 2024-11-09T00:23:56,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/515f5811552847af80b2fea90a84a363 2024-11-09T00:23:56,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/5c1967e272b645e3b95831fc1f3503ae 2024-11-09T00:23:56,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a6c81bdbab864975b23fe931b1cdb520, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff80e35074944389a344b287cd1e4769, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f54bae01c01b4a7a9d45675da0875251, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/76672d798c9a436b8314a186f281560a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/37b94d7d37ba4bd6bf5ce218040c6a75, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a03044e7aba441c19b95a9a2e8621330, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e09d57d2ccad4e77a33aefd547728a6b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1d02aa58d1ac4794b527c4717eb1ee30, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5] to archive 2024-11-09T00:23:56,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:56,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a6c81bdbab864975b23fe931b1cdb520 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a6c81bdbab864975b23fe931b1cdb520 2024-11-09T00:23:56,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/cc82a7801c3a4046acb2b06636b802c7 2024-11-09T00:23:56,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff53084cccd940fe8aeba15b6b456451 2024-11-09T00:23:56,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff80e35074944389a344b287cd1e4769 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/ff80e35074944389a344b287cd1e4769 2024-11-09T00:23:56,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/7217ce12d8b34139af64647f47ce713e 2024-11-09T00:23:56,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/94fbc9d701f54646be10ff65919fe371 2024-11-09T00:23:56,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f54bae01c01b4a7a9d45675da0875251 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f54bae01c01b4a7a9d45675da0875251 2024-11-09T00:23:56,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/42b94500d84a4ab29250c86ad45cfe3d 2024-11-09T00:23:56,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e0e19e1fd3754fd89b83156353a475f0 2024-11-09T00:23:56,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/76672d798c9a436b8314a186f281560a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/76672d798c9a436b8314a186f281560a 2024-11-09T00:23:56,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/634bc848a0e5472ca2b7fe3f5b078ad3 2024-11-09T00:23:56,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/208aa12aba58406091c67b7dc8a6489e 2024-11-09T00:23:56,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/37b94d7d37ba4bd6bf5ce218040c6a75 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/37b94d7d37ba4bd6bf5ce218040c6a75 2024-11-09T00:23:56,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/375dde6872d844f3805cf91c28aa6bc4 2024-11-09T00:23:56,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1e05c6bbbd19475eaff97bdfc57d8adc 2024-11-09T00:23:56,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a03044e7aba441c19b95a9a2e8621330 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a03044e7aba441c19b95a9a2e8621330 2024-11-09T00:23:56,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/281f4b2b9fd84187b3308aeb1bcdaa7e 2024-11-09T00:23:56,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/74680bd6d50a427bbe1ad24c8f9a210e 2024-11-09T00:23:56,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e09d57d2ccad4e77a33aefd547728a6b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/e09d57d2ccad4e77a33aefd547728a6b 2024-11-09T00:23:56,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/36c0dd40fe1a42d385ef1052e6c490da 2024-11-09T00:23:56,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/a98951d3ddf44357a5451dbeb9b87ef4 2024-11-09T00:23:56,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1d02aa58d1ac4794b527c4717eb1ee30 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/1d02aa58d1ac4794b527c4717eb1ee30 2024-11-09T00:23:56,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/d5f4ef7bcced402fbd54e0ee1b217477 2024-11-09T00:23:56,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/28750ec5c54543e991c24136008c63be 2024-11-09T00:23:56,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/33a5bfdf624942f7b9c256355680c7c5 2024-11-09T00:23:56,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ac4888f42a8d43449e5681b9504ba76e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e220b56d448a4d40bf77d7efaef4d15e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75385e8c82bd440dbc90d263a947ef34, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/317019681dbe427498542ca9e7427ca8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2da82675053246fc9c43c8bc4b2f66dc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/a310cacfe30c4b748790334ee66163d1, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2d259c0d2e3c462386e079af2ec976b0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ffc21e15319d4304a0fb6347e0a57ddb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d] to archive 2024-11-09T00:23:56,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:23:56,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ac4888f42a8d43449e5681b9504ba76e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ac4888f42a8d43449e5681b9504ba76e 2024-11-09T00:23:56,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/9893b290885d403d99b949e23919f0f2 2024-11-09T00:23:56,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/508f962164e24d10b70ce6dcfebe5fb7 2024-11-09T00:23:56,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e220b56d448a4d40bf77d7efaef4d15e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e220b56d448a4d40bf77d7efaef4d15e 2024-11-09T00:23:56,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/593c24b49b684ce59750a90719426131 2024-11-09T00:23:56,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/c077c325038348938ed1b9c126c10a71 2024-11-09T00:23:56,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75385e8c82bd440dbc90d263a947ef34 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75385e8c82bd440dbc90d263a947ef34 2024-11-09T00:23:56,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/75d1058b46cc40789269319f8117f2e7 2024-11-09T00:23:56,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/f5db29ae4a894e899be2b51fa9dc872f 2024-11-09T00:23:56,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/317019681dbe427498542ca9e7427ca8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/317019681dbe427498542ca9e7427ca8 2024-11-09T00:23:56,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/8dcf0e1c8ca243f8afaaaf83030bfe91 2024-11-09T00:23:56,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/55de899c584547e0baf681e826ce7e51 2024-11-09T00:23:56,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2da82675053246fc9c43c8bc4b2f66dc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2da82675053246fc9c43c8bc4b2f66dc 2024-11-09T00:23:56,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/73d6b881096a42cda4052cf3710741b7 2024-11-09T00:23:56,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/6e6147503faf4fe79ea154fcb93775dd 2024-11-09T00:23:56,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/a310cacfe30c4b748790334ee66163d1 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/a310cacfe30c4b748790334ee66163d1 2024-11-09T00:23:56,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/5f6acce87fe74b8187a1d12eacc31cf0 2024-11-09T00:23:56,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/427768a801c54fa49dd0949d65c45594 2024-11-09T00:23:56,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2d259c0d2e3c462386e079af2ec976b0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/2d259c0d2e3c462386e079af2ec976b0 2024-11-09T00:23:56,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/68f3812b1dce4b18827387d8a0a436de 2024-11-09T00:23:56,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d1ac40e9d31b4f8a9c7dc2884b024545 2024-11-09T00:23:56,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ffc21e15319d4304a0fb6347e0a57ddb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/ffc21e15319d4304a0fb6347e0a57ddb 2024-11-09T00:23:56,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/dcb2aadc155a4e75ae499e887129e836 2024-11-09T00:23:56,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d632f7c0092d4bec8d2af54b55c565e7 2024-11-09T00:23:56,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/e28b34762dac488ca4c4b1e4cac6d22d 2024-11-09T00:23:56,836 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=1 2024-11-09T00:23:56,837 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. 2024-11-09T00:23:56,837 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 2edb930c55ccfc4a4303c1efb2c810a9: Waiting for close lock at 1731111835488Running coprocessor pre-close hooks at 1731111835488Disabling compacts and flushes for region at 1731111835488Disabling writes for close at 1731111835488Obtaining lock to block concurrent updates at 1731111835488Preparing flush snapshotting stores in 2edb930c55ccfc4a4303c1efb2c810a9 at 1731111835488Finished memstore snapshotting TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731111835489 (+1 ms)Flushing stores of TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9. at 1731111835489Flushing 2edb930c55ccfc4a4303c1efb2c810a9/A: creating writer at 1731111835489Flushing 2edb930c55ccfc4a4303c1efb2c810a9/A: appending metadata at 1731111835492 (+3 ms)Flushing 2edb930c55ccfc4a4303c1efb2c810a9/A: closing flushed file at 1731111835492Flushing 2edb930c55ccfc4a4303c1efb2c810a9/B: creating writer at 1731111835905 (+413 ms)Flushing 2edb930c55ccfc4a4303c1efb2c810a9/B: appending metadata at 1731111835908 (+3 ms)Flushing 2edb930c55ccfc4a4303c1efb2c810a9/B: closing flushed file at 1731111835908Flushing 2edb930c55ccfc4a4303c1efb2c810a9/C: creating writer at 1731111836325 (+417 ms)Flushing 2edb930c55ccfc4a4303c1efb2c810a9/C: appending metadata at 1731111836329 (+4 ms)Flushing 2edb930c55ccfc4a4303c1efb2c810a9/C: closing flushed file at 1731111836329Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71cc8789: reopening flushed file at 1731111836742 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@317d96ea: reopening flushed file at 1731111836747 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1724ae36: reopening flushed file at 1731111836750 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2edb930c55ccfc4a4303c1efb2c810a9 in 1267ms, sequenceid=396, compaction requested=false at 1731111836755 (+5 ms)Writing region close event to WAL at 1731111836833 (+78 ms)Running coprocessor post-close hooks at 1731111836837 (+4 ms)Closed at 1731111836837 2024-11-09T00:23:56,839 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:56,840 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2edb930c55ccfc4a4303c1efb2c810a9, regionState=CLOSED 2024-11-09T00:23:56,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:56,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T00:23:56,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 2edb930c55ccfc4a4303c1efb2c810a9, server=ef986b38239e,35861,1731111692607 in 1.5130 sec 2024-11-09T00:23:56,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-09T00:23:56,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2edb930c55ccfc4a4303c1efb2c810a9, UNASSIGN in 1.5180 sec 2024-11-09T00:23:56,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T00:23:56,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-09T00:23:56,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111836854"}]},"ts":"1731111836854"} 2024-11-09T00:23:56,856 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:23:56,856 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:23:56,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-09T00:23:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T00:23:57,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:57,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:23:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,478 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T00:23:57,479 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,484 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:57,489 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/recovered.edits] 2024-11-09T00:23:57,493 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c11c1d40b1904f0a9b55e52a26461660 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/c11c1d40b1904f0a9b55e52a26461660 2024-11-09T00:23:57,494 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/f95797df8fbb444981cf1a6c9f20ebff to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/A/f95797df8fbb444981cf1a6c9f20ebff 2024-11-09T00:23:57,496 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/3aa6e0a28e81422b85028fc94436d0f0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/3aa6e0a28e81422b85028fc94436d0f0 2024-11-09T00:23:57,497 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f7bdccaf51e04445829721c8075a7428 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/B/f7bdccaf51e04445829721c8075a7428 2024-11-09T00:23:57,498 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/65839deca69141d49e01866a2839688a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/65839deca69141d49e01866a2839688a 2024-11-09T00:23:57,499 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d00430c0195a4559b8aed84e276cef88 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/C/d00430c0195a4559b8aed84e276cef88 2024-11-09T00:23:57,501 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/recovered.edits/399.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9/recovered.edits/399.seqid 2024-11-09T00:23:57,502 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/2edb930c55ccfc4a4303c1efb2c810a9 2024-11-09T00:23:57,502 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:23:57,504 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,506 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:23:57,508 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:23:57,509 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,509 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:23:57,509 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111837509"}]},"ts":"9223372036854775807"} 2024-11-09T00:23:57,511 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:23:57,511 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2edb930c55ccfc4a4303c1efb2c810a9, NAME => 'TestAcidGuarantees,,1731111808957.2edb930c55ccfc4a4303c1efb2c810a9.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:23:57,511 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:23:57,511 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111837511"}]},"ts":"9223372036854775807"} 2024-11-09T00:23:57,513 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:23:57,514 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-09T00:23:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T00:23:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:23:57,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:57,596 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 241), OpenFileDescriptor=451 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=559 (was 612), ProcessCount=11 (was 11), AvailableMemoryMB=4267 (was 3031) - AvailableMemoryMB LEAK? - 2024-11-09T00:23:57,607 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=559, ProcessCount=11, AvailableMemoryMB=4266 2024-11-09T00:23:57,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:23:57,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:23:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:57,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T00:23:57,611 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:57,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-09T00:23:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:57,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T00:23:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742342_1518 (size=959) 2024-11-09T00:23:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:58,021 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b 2024-11-09T00:23:58,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742343_1519 (size=53) 2024-11-09T00:23:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:58,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:58,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c5b55e77595e7e32ac3fc5b9600e8a84, disabling compactions & flushes 2024-11-09T00:23:58,428 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. after waiting 0 ms 2024-11-09T00:23:58,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,428 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c5b55e77595e7e32ac3fc5b9600e8a84: Waiting for close lock at 1731111838428Disabling compacts and flushes for region at 1731111838428Disabling writes for close at 1731111838428Writing region close event to WAL at 1731111838428Closed at 1731111838428 2024-11-09T00:23:58,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T00:23:58,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731111838430"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731111838430"}]},"ts":"1731111838430"} 2024-11-09T00:23:58,431 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T00:23:58,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T00:23:58,432 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111838432"}]},"ts":"1731111838432"} 2024-11-09T00:23:58,434 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T00:23:58,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, ASSIGN}] 2024-11-09T00:23:58,435 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, ASSIGN 2024-11-09T00:23:58,436 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, ASSIGN; state=OFFLINE, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=false 2024-11-09T00:23:58,586 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:58,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, ASSIGN because future has completed 2024-11-09T00:23:58,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:23:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:58,746 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,746 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:23:58,747 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,747 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:23:58,748 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,748 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,750 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,751 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName A 2024-11-09T00:23:58,752 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:58,752 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:58,752 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,753 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName B 2024-11-09T00:23:58,753 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:58,754 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:58,754 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,754 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName C 2024-11-09T00:23:58,755 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:23:58,755 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:23:58,755 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,755 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,756 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,756 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,756 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,757 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:23:58,758 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,759 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T00:23:58,759 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened c5b55e77595e7e32ac3fc5b9600e8a84; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61129019, jitterRate=-0.08910663425922394}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:23:58,759 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:23:58,760 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for c5b55e77595e7e32ac3fc5b9600e8a84: Running coprocessor pre-open hook at 1731111838748Writing region info on filesystem at 1731111838748Initializing all the Stores at 1731111838749 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111838749Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111838750 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111838750Cleaning up temporary data from old regions at 1731111838756 (+6 ms)Running coprocessor post-open hooks at 1731111838759 (+3 ms)Region opened successfully at 1731111838760 (+1 ms) 2024-11-09T00:23:58,760 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., pid=136, masterSystemTime=1731111838740 2024-11-09T00:23:58,762 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,762 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:23:58,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=OPEN, openSeqNum=2, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:23:58,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:23:58,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-09T00:23:58,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 in 176 msec 2024-11-09T00:23:58,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-09T00:23:58,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, ASSIGN in 332 msec 2024-11-09T00:23:58,768 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T00:23:58,768 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111838768"}]},"ts":"1731111838768"} 2024-11-09T00:23:58,770 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T00:23:58,770 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T00:23:58,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-09T00:23:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-09T00:23:59,755 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:23:59,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:59,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:23:59,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:23:59,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:23:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:23:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:23:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ab5178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:23:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:23:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:23:59,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:23:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17547ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:23:59,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:23:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:23:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:23:59,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:23:59,767 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T00:23:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ef986b38239e,38689,1731111691424 2024-11-09T00:23:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@bd865ac 2024-11-09T00:23:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T00:23:59,769 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T00:23:59,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T00:23:59,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T00:23:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T00:23:59,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742344_1520 (size=995) 2024-11-09T00:24:00,119 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:24:00,184 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T00:24:00,184 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T00:24:00,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:24:00,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE}] 2024-11-09T00:24:00,188 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE 2024-11-09T00:24:00,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE because future has completed 2024-11-09T00:24:00,190 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:24:00,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:24:00,343 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,343 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:24:00,343 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing c5b55e77595e7e32ac3fc5b9600e8a84, disabling compactions & flushes 2024-11-09T00:24:00,343 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,343 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,343 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. after waiting 0 ms 2024-11-09T00:24:00,343 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,346 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T00:24:00,347 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,347 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for c5b55e77595e7e32ac3fc5b9600e8a84: Waiting for close lock at 1731111840343Running coprocessor pre-close hooks at 1731111840343Disabling compacts and flushes for region at 1731111840343Disabling writes for close at 1731111840343Writing region close event to WAL at 1731111840343Running coprocessor post-close hooks at 1731111840347 (+4 ms)Closed at 1731111840347 2024-11-09T00:24:00,347 WARN [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: c5b55e77595e7e32ac3fc5b9600e8a84 to self. 2024-11-09T00:24:00,349 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,349 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=CLOSED 2024-11-09T00:24:00,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:24:00,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T00:24:00,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 in 161 msec 2024-11-09T00:24:00,353 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE; state=CLOSED, location=ef986b38239e,35861,1731111692607; forceNewPlan=false, retain=true 2024-11-09T00:24:00,504 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=OPENING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE because future has completed 2024-11-09T00:24:00,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:24:00,660 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,660 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} 2024-11-09T00:24:00,660 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,660 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T00:24:00,660 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,660 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,662 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,662 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName A 2024-11-09T00:24:00,663 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:00,664 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:24:00,664 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,665 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName B 2024-11-09T00:24:00,665 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:00,665 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:24:00,666 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,666 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5b55e77595e7e32ac3fc5b9600e8a84 columnFamilyName C 2024-11-09T00:24:00,666 DEBUG [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:00,667 INFO [StoreOpener-c5b55e77595e7e32ac3fc5b9600e8a84-1 {}] regionserver.HStore(327): Store=c5b55e77595e7e32ac3fc5b9600e8a84/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T00:24:00,667 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,667 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,668 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,669 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,669 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,670 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T00:24:00,671 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,671 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened c5b55e77595e7e32ac3fc5b9600e8a84; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61710217, jitterRate=-0.08044610917568207}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T00:24:00,672 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,672 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for c5b55e77595e7e32ac3fc5b9600e8a84: Running coprocessor pre-open hook at 1731111840660Writing region info on filesystem at 1731111840660Initializing all the Stores at 1731111840661 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111840661Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111840661Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731111840661Cleaning up temporary data from old regions at 1731111840669 (+8 ms)Running coprocessor post-open hooks at 1731111840672 (+3 ms)Region opened successfully at 1731111840672 2024-11-09T00:24:00,673 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., pid=141, masterSystemTime=1731111840657 2024-11-09T00:24:00,675 DEBUG [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,675 INFO [RS_OPEN_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=OPEN, openSeqNum=5, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:24:00,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-09T00:24:00,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 in 172 msec 2024-11-09T00:24:00,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-09T00:24:00,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, REOPEN/MOVE in 491 msec 2024-11-09T00:24:00,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-09T00:24:00,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 495 msec 2024-11-09T00:24:00,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-09T00:24:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T00:24:00,686 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:00,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,687 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df6f677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da0a0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,691 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479d9248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dff1b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bac2785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,696 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d22626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12315ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a539037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,706 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d572fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f216685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,713 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ebf5d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1424d7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df1119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d354b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59838687, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7348dda0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,724 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a53c5ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ebfa86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef986b38239e,38689,-1 for getting cluster id 2024-11-09T00:24:00,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T00:24:00,727 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b39df24e-75e2-4c08-8f69-2ef0b1a146a3' 2024-11-09T00:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T00:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b39df24e-75e2-4c08-8f69-2ef0b1a146a3" 2024-11-09T00:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bddbd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef986b38239e,38689,-1] 2024-11-09T00:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T00:24:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:00,728 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T00:24:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fda072b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T00:24:00,729 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,730 DEBUG [Thread-2322 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,731 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,732 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,733 DEBUG [Thread-2341 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,733 DEBUG [Thread-2341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,735 DEBUG [Thread-2324 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-09T00:24:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:00,737 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,737 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,738 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:00,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:00,738 DEBUG [Thread-2326 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,744 DEBUG [Thread-2335 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:00,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:00,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T00:24:00,745 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,748 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T00:24:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef986b38239e,35861,1731111692607, seqNum=-1] 2024-11-09T00:24:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T00:24:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T00:24:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5] 2024-11-09T00:24:00,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110957d043110a9c4ada8869c90f7c44412b_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111840744/Put/seqid=0 2024-11-09T00:24:00,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742345_1521 (size=9714) 2024-11-09T00:24:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111900800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111900802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111900801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111900802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111900803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:00,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:00,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:00,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111900915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111900915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111900915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111900915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111900915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111901124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111901124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111901127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111901127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111901127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,185 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:01,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110957d043110a9c4ada8869c90f7c44412b_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957d043110a9c4ada8869c90f7c44412b_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:01,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:01,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d is 175, key is test_row_0/A:col10/1731111840744/Put/seqid=0 2024-11-09T00:24:01,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742346_1522 (size=22361) 2024-11-09T00:24:01,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d 2024-11-09T00:24:01,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/bba72e60a60b40ab97988312b73d0fc4 is 50, key is test_row_0/B:col10/1731111840744/Put/seqid=0 2024-11-09T00:24:01,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742347_1523 (size=9657) 2024-11-09T00:24:01,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/bba72e60a60b40ab97988312b73d0fc4 2024-11-09T00:24:01,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/20a0557684fb4bc2a5c658ddafdc6860 is 50, key is test_row_0/C:col10/1731111840744/Put/seqid=0 2024-11-09T00:24:01,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742348_1524 (size=9657) 2024-11-09T00:24:01,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:01,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:01,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:01,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:02,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/20a0557684fb4bc2a5c658ddafdc6860 2024-11-09T00:24:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d 2024-11-09T00:24:02,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d, entries=100, sequenceid=16, filesize=21.8 K 2024-11-09T00:24:02,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/bba72e60a60b40ab97988312b73d0fc4 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4 2024-11-09T00:24:02,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4, entries=100, sequenceid=16, filesize=9.4 K 2024-11-09T00:24:02,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/20a0557684fb4bc2a5c658ddafdc6860 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860 2024-11-09T00:24:02,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860, entries=100, sequenceid=16, filesize=9.4 K 2024-11-09T00:24:02,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1343ms, sequenceid=16, compaction requested=false 2024-11-09T00:24:02,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:02,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-09T00:24:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:02,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:24:02,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090e35bf2c01634ac0b5ce96ae37135296_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111840769/Put/seqid=0 2024-11-09T00:24:02,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742349_1525 (size=12154) 2024-11-09T00:24:02,366 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T00:24:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:02,541 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090e35bf2c01634ac0b5ce96ae37135296_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e35bf2c01634ac0b5ce96ae37135296_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bfb6fd324d13425e946d6af4bedeedc8, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bfb6fd324d13425e946d6af4bedeedc8 is 175, key is test_row_0/A:col10/1731111840769/Put/seqid=0 2024-11-09T00:24:02,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742350_1526 (size=30955) 2024-11-09T00:24:02,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bfb6fd324d13425e946d6af4bedeedc8 2024-11-09T00:24:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/842cdbc570074e219aaea9062ce2c417 is 50, key is test_row_0/B:col10/1731111840769/Put/seqid=0 2024-11-09T00:24:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742351_1527 (size=12001) 2024-11-09T00:24:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:02,960 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/842cdbc570074e219aaea9062ce2c417 2024-11-09T00:24:02,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:02,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111902973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111902974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:02,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111902978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/b2730786000549cdb728bdaa4323943a is 50, key is test_row_0/C:col10/1731111840769/Put/seqid=0 2024-11-09T00:24:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111902980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111902981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742352_1528 (size=12001) 2024-11-09T00:24:03,006 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/b2730786000549cdb728bdaa4323943a 2024-11-09T00:24:03,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/bfb6fd324d13425e946d6af4bedeedc8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8 2024-11-09T00:24:03,067 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T00:24:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/842cdbc570074e219aaea9062ce2c417 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417 2024-11-09T00:24:03,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111903084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111903085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,094 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T00:24:03,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/b2730786000549cdb728bdaa4323943a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a 2024-11-09T00:24:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111903095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,100 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T00:24:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111903095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c5b55e77595e7e32ac3fc5b9600e8a84 in 971ms, sequenceid=41, compaction requested=false 2024-11-09T00:24:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-09T00:24:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-09T00:24:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:03,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-09T00:24:03,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:03,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-09T00:24:03,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:24:03,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.3740 sec 2024-11-09T00:24:03,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110915028aa556304e56a409e5814327943d_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:03,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742353_1529 (size=14594) 2024-11-09T00:24:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111903229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111903295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111903295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111903304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111903304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111903344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:03,557 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110915028aa556304e56a409e5814327943d_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110915028aa556304e56a409e5814327943d_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:03,558 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/1901e88e2954464e90f0fbf802588fcf, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:03,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/1901e88e2954464e90f0fbf802588fcf is 175, key is test_row_0/A:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111903555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742354_1530 (size=39549) 2024-11-09T00:24:03,570 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/1901e88e2954464e90f0fbf802588fcf 2024-11-09T00:24:03,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/442f27f3bd624f65b7d5ff50ecd18566 is 50, key is test_row_0/B:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:03,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742355_1531 (size=12001) 2024-11-09T00:24:03,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/442f27f3bd624f65b7d5ff50ecd18566 2024-11-09T00:24:03,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b62c7334e1042989f72b1d82de8bff6 is 50, key is test_row_0/C:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111903604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111903615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111903616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111903620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:03,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742356_1532 (size=12001) 2024-11-09T00:24:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111903864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b62c7334e1042989f72b1d82de8bff6 2024-11-09T00:24:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/1901e88e2954464e90f0fbf802588fcf as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf 2024-11-09T00:24:04,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf, entries=200, sequenceid=53, filesize=38.6 K 2024-11-09T00:24:04,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/442f27f3bd624f65b7d5ff50ecd18566 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566 2024-11-09T00:24:04,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T00:24:04,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b62c7334e1042989f72b1d82de8bff6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6 2024-11-09T00:24:04,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T00:24:04,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c5b55e77595e7e32ac3fc5b9600e8a84 in 929ms, sequenceid=53, compaction requested=true 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:04,037 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:04,037 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:04,038 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:04,038 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:04,038 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=90.7 K 2024-11-09T00:24:04,038 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=32.9 K 2024-11-09T00:24:04,038 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf] 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting bffa341cc1b24ca1a4f2bedbe7ea3f2d, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731111840741 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting bba72e60a60b40ab97988312b73d0fc4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731111840741 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfb6fd324d13425e946d6af4bedeedc8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111840769 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 842cdbc570074e219aaea9062ce2c417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111840769 2024-11-09T00:24:04,038 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1901e88e2954464e90f0fbf802588fcf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842969 2024-11-09T00:24:04,039 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 442f27f3bd624f65b7d5ff50ecd18566, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842973 2024-11-09T00:24:04,042 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:04,044 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110952412f3617584d03ae2ea1054d3e73fc_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:04,044 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:04,044 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/d7084427eb10467ea9ebbbd35582fab2 is 50, key is test_row_0/B:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:04,045 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110952412f3617584d03ae2ea1054d3e73fc_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:04,046 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110952412f3617584d03ae2ea1054d3e73fc_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:04,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742357_1533 (size=12104) 2024-11-09T00:24:04,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742358_1534 (size=4469) 2024-11-09T00:24:04,052 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/d7084427eb10467ea9ebbbd35582fab2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d7084427eb10467ea9ebbbd35582fab2 2024-11-09T00:24:04,056 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into d7084427eb10467ea9ebbbd35582fab2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:04,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:04,056 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=13, startTime=1731111844037; duration=0sec 2024-11-09T00:24:04,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:04,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:04,056 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:04,057 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:04,057 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:04,057 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:04,057 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=32.9 K 2024-11-09T00:24:04,057 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a0557684fb4bc2a5c658ddafdc6860, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731111840741 2024-11-09T00:24:04,057 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting b2730786000549cdb728bdaa4323943a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731111840769 2024-11-09T00:24:04,058 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b62c7334e1042989f72b1d82de8bff6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842973 2024-11-09T00:24:04,064 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:04,065 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/2a40138eb49b4ea1b23e13e4fa6de74a is 50, key is test_row_0/C:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:04,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742359_1535 (size=12104) 2024-11-09T00:24:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:04,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:04,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:24:04,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094e1f83606ed248f58d834c988db00b26_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111844116/Put/seqid=0 2024-11-09T00:24:04,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742360_1536 (size=14594) 2024-11-09T00:24:04,126 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:04,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094e1f83606ed248f58d834c988db00b26_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094e1f83606ed248f58d834c988db00b26_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:04,130 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4022cc01ad1a4ae6ae33a5058603d845, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:04,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4022cc01ad1a4ae6ae33a5058603d845 is 175, key is test_row_0/A:col10/1731111844116/Put/seqid=0 2024-11-09T00:24:04,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742361_1537 (size=39549) 2024-11-09T00:24:04,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111904134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111904135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111904140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111904140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111904244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111904244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111904254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111904254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111904374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,449 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:04,450 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/8fde57b1f689446088c8d688d17a8eee is 175, key is test_row_0/A:col10/1731111842973/Put/seqid=0 2024-11-09T00:24:04,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742362_1538 (size=31058) 2024-11-09T00:24:04,455 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/8fde57b1f689446088c8d688d17a8eee as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee 2024-11-09T00:24:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111904454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111904454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,459 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into 8fde57b1f689446088c8d688d17a8eee(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:04,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:04,459 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=13, startTime=1731111844037; duration=0sec 2024-11-09T00:24:04,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:04,459 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111904465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111904465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/2a40138eb49b4ea1b23e13e4fa6de74a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2a40138eb49b4ea1b23e13e4fa6de74a 2024-11-09T00:24:04,483 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 2a40138eb49b4ea1b23e13e4fa6de74a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:04,483 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:04,483 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=13, startTime=1731111844037; duration=0sec 2024-11-09T00:24:04,483 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:04,483 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:04,535 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4022cc01ad1a4ae6ae33a5058603d845 2024-11-09T00:24:04,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0d1d503cd3b5419786d8b570f4b25e7a is 50, key is test_row_0/B:col10/1731111844116/Put/seqid=0 2024-11-09T00:24:04,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742363_1539 (size=12001) 2024-11-09T00:24:04,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111904764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111904765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111904774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111904774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-09T00:24:04,914 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:04,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-09T00:24:04,916 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T00:24:04,917 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:04,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:04,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0d1d503cd3b5419786d8b570f4b25e7a 2024-11-09T00:24:04,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/51151ca3cb9a48ebbab3806b73c68d03 is 50, key is test_row_0/C:col10/1731111844116/Put/seqid=0 2024-11-09T00:24:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742364_1540 (size=12001) 2024-11-09T00:24:04,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/51151ca3cb9a48ebbab3806b73c68d03 2024-11-09T00:24:04,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4022cc01ad1a4ae6ae33a5058603d845 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845 2024-11-09T00:24:04,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845, entries=200, sequenceid=79, filesize=38.6 K 2024-11-09T00:24:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0d1d503cd3b5419786d8b570f4b25e7a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a 2024-11-09T00:24:04,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T00:24:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/51151ca3cb9a48ebbab3806b73c68d03 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03 2024-11-09T00:24:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T00:24:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for c5b55e77595e7e32ac3fc5b9600e8a84 in 873ms, sequenceid=79, compaction requested=false 2024-11-09T00:24:04,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T00:24:05,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-09T00:24:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:05,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T00:24:05,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6f4ebb34c504c029e805a705afbfe94_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111844139/Put/seqid=0 2024-11-09T00:24:05,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742365_1541 (size=12154) 2024-11-09T00:24:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T00:24:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111905310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111905311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111905312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111905312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111905394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,397 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111905424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111905425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111905425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111905425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:05,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e6f4ebb34c504c029e805a705afbfe94_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6f4ebb34c504c029e805a705afbfe94_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:05,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/7ae7882b8ff54b828c7900cb24adadff, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:05,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/7ae7882b8ff54b828c7900cb24adadff is 175, key is test_row_0/A:col10/1731111844139/Put/seqid=0 2024-11-09T00:24:05,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742366_1542 (size=30955) 2024-11-09T00:24:05,496 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/7ae7882b8ff54b828c7900cb24adadff 2024-11-09T00:24:05,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/a24b62127cd74712bf8386ea621ea789 is 50, key is test_row_0/B:col10/1731111844139/Put/seqid=0 2024-11-09T00:24:05,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742367_1543 (size=12001) 2024-11-09T00:24:05,514 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/a24b62127cd74712bf8386ea621ea789 2024-11-09T00:24:05,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b7a21e7625c4bcdbee1e5266681b091 is 50, key is test_row_0/C:col10/1731111844139/Put/seqid=0 2024-11-09T00:24:05,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742368_1544 (size=12001) 2024-11-09T00:24:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T00:24:05,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111905635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111905635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111905635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111905635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,936 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b7a21e7625c4bcdbee1e5266681b091 2024-11-09T00:24:05,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/7ae7882b8ff54b828c7900cb24adadff as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff 2024-11-09T00:24:05,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111905944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111905945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff, entries=150, sequenceid=93, filesize=30.2 K 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111905945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111905945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/a24b62127cd74712bf8386ea621ea789 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:05,950 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T00:24:05,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/4b7a21e7625c4bcdbee1e5266681b091 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091 2024-11-09T00:24:05,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T00:24:05,955 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for c5b55e77595e7e32ac3fc5b9600e8a84 in 886ms, sequenceid=93, compaction requested=true 2024-11-09T00:24:05,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:05,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:05,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-09T00:24:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-09T00:24:05,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-09T00:24:05,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-11-09T00:24:05,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.0430 sec 2024-11-09T00:24:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-09T00:24:06,055 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:06,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-09T00:24:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T00:24:06,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:06,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:06,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T00:24:06,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T00:24:06,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:06,209 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T00:24:06,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dc7c5140c7324f1baaebca15db89aa99_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111845310/Put/seqid=0 2024-11-09T00:24:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742369_1545 (size=12154) 2024-11-09T00:24:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:06,229 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dc7c5140c7324f1baaebca15db89aa99_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc7c5140c7324f1baaebca15db89aa99_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/085c969de59b48818fa56ca0c116e680, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/085c969de59b48818fa56ca0c116e680 is 175, key is test_row_0/A:col10/1731111845310/Put/seqid=0 2024-11-09T00:24:06,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742370_1546 (size=30955) 2024-11-09T00:24:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T00:24:06,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111906463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111906495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111906495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111906495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111906604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111906604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111906604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111906604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,637 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/085c969de59b48818fa56ca0c116e680 2024-11-09T00:24:06,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 is 50, key is test_row_0/B:col10/1731111845310/Put/seqid=0 2024-11-09T00:24:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742371_1547 (size=12001) 2024-11-09T00:24:06,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 2024-11-09T00:24:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/ad2b07566f794137b441ddf02e9379cc is 50, key is test_row_0/C:col10/1731111845310/Put/seqid=0 2024-11-09T00:24:06,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742372_1548 (size=12001) 2024-11-09T00:24:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T00:24:06,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111906814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111906815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111906815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111906815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,052 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/ad2b07566f794137b441ddf02e9379cc 2024-11-09T00:24:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/085c969de59b48818fa56ca0c116e680 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680 2024-11-09T00:24:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680, entries=150, sequenceid=118, filesize=30.2 K 2024-11-09T00:24:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 2024-11-09T00:24:07,063 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T00:24:07,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/ad2b07566f794137b441ddf02e9379cc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc 2024-11-09T00:24:07,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T00:24:07,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for c5b55e77595e7e32ac3fc5b9600e8a84 in 857ms, sequenceid=118, compaction requested=true 2024-11-09T00:24:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-09T00:24:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-09T00:24:07,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-09T00:24:07,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-09T00:24:07,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.0140 sec 2024-11-09T00:24:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:07,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T00:24:07,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110999f9c77931534a5586ec6d8328d27c1f_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:07,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742373_1549 (size=17184) 2024-11-09T00:24:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111907156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111907161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111907161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111907161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T00:24:07,195 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:07,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-09T00:24:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:07,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:07,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:07,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:07,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111907264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111907274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111907274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111907275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:07,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111907404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,407 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111907474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111907485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111907485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111907485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:07,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:07,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110999f9c77931534a5586ec6d8328d27c1f_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110999f9c77931534a5586ec6d8328d27c1f_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:07,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/b3ebfff814d3413294165f464aaea281, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:07,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/b3ebfff814d3413294165f464aaea281 is 175, key is test_row_0/A:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:07,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742374_1550 (size=48289) 2024-11-09T00:24:07,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/b3ebfff814d3413294165f464aaea281 2024-11-09T00:24:07,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0f366def2e4643e9af124c4a6c200145 is 50, key is test_row_0/B:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742375_1551 (size=12101) 2024-11-09T00:24:07,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0f366def2e4643e9af124c4a6c200145 2024-11-09T00:24:07,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/2181e99d1a8648968105f560eebcecc8 is 50, key is test_row_0/C:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:07,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742376_1552 (size=12101) 2024-11-09T00:24:07,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111907785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111907795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111907795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111907797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:07,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:07,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:07,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:07,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:07,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/2181e99d1a8648968105f560eebcecc8 2024-11-09T00:24:08,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/b3ebfff814d3413294165f464aaea281 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281 2024-11-09T00:24:08,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281, entries=250, sequenceid=132, filesize=47.2 K 2024-11-09T00:24:08,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0f366def2e4643e9af124c4a6c200145 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145 2024-11-09T00:24:08,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T00:24:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/2181e99d1a8648968105f560eebcecc8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8 2024-11-09T00:24:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T00:24:08,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for c5b55e77595e7e32ac3fc5b9600e8a84 in 901ms, sequenceid=132, compaction requested=true 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:08,027 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T00:24:08,027 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:08,029 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,029 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d7084427eb10467ea9ebbbd35582fab2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=58.8 K 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 180806 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:08,029 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,029 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=176.6 K 2024-11-09T00:24:08,029 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281] 2024-11-09T00:24:08,029 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d7084427eb10467ea9ebbbd35582fab2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842973 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fde57b1f689446088c8d688d17a8eee, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842973 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d1d503cd3b5419786d8b570f4b25e7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731111843219 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4022cc01ad1a4ae6ae33a5058603d845, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731111843219 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a24b62127cd74712bf8386ea621ea789, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731111844133 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ae7882b8ff54b828c7900cb24adadff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731111844133 2024-11-09T00:24:08,030 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f46bb9d3c34ee7ac575c7d4e4d08b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731111845310 2024-11-09T00:24:08,031 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085c969de59b48818fa56ca0c116e680, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731111845310 2024-11-09T00:24:08,031 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f366def2e4643e9af124c4a6c200145, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846467 2024-11-09T00:24:08,031 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ebfff814d3413294165f464aaea281, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846462 2024-11-09T00:24:08,039 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:08,042 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:08,042 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/9517e86c9e194f8aa3dff9317a576cb8 is 50, key is test_row_0/B:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:08,043 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ecfa44e105cc4c31b445b3026d6e98c0_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:08,047 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ecfa44e105cc4c31b445b3026d6e98c0_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:08,047 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ecfa44e105cc4c31b445b3026d6e98c0_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:08,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742377_1553 (size=12375) 2024-11-09T00:24:08,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742378_1554 (size=4469) 2024-11-09T00:24:08,065 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#467 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:08,065 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c857b0e574f44be493115442862a2692 is 175, key is test_row_0/A:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742379_1555 (size=31329) 2024-11-09T00:24:08,078 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c857b0e574f44be493115442862a2692 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692 2024-11-09T00:24:08,085 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into c857b0e574f44be493115442862a2692(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:08,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:08,085 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=11, startTime=1731111848027; duration=0sec 2024-11-09T00:24:08,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:08,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:08,085 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T00:24:08,088 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T00:24:08,088 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:08,088 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,088 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2a40138eb49b4ea1b23e13e4fa6de74a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=58.8 K 2024-11-09T00:24:08,089 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a40138eb49b4ea1b23e13e4fa6de74a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731111842973 2024-11-09T00:24:08,089 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51151ca3cb9a48ebbab3806b73c68d03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731111843219 2024-11-09T00:24:08,089 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7a21e7625c4bcdbee1e5266681b091, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731111844133 2024-11-09T00:24:08,089 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2b07566f794137b441ddf02e9379cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731111845310 2024-11-09T00:24:08,090 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2181e99d1a8648968105f560eebcecc8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846467 2024-11-09T00:24:08,103 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#469 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:08,103 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/5f4f0fe56b3d4899b704e616dff23d91 is 50, key is test_row_0/C:col10/1731111847125/Put/seqid=0 2024-11-09T00:24:08,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742380_1556 (size=12375) 2024-11-09T00:24:08,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T00:24:08,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,121 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T00:24:08,125 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/5f4f0fe56b3d4899b704e616dff23d91 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/5f4f0fe56b3d4899b704e616dff23d91 2024-11-09T00:24:08,130 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 5f4f0fe56b3d4899b704e616dff23d91(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:08,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:08,130 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=11, startTime=1731111848027; duration=0sec 2024-11-09T00:24:08,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:08,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:08,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094013f2718fb04972a9042b20a37d920c_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111847160/Put/seqid=0 2024-11-09T00:24:08,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742381_1557 (size=12304) 2024-11-09T00:24:08,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:08,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094013f2718fb04972a9042b20a37d920c_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094013f2718fb04972a9042b20a37d920c_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/165d9ece6b4c46a4ac59f57204a650cf, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/165d9ece6b4c46a4ac59f57204a650cf is 175, key is test_row_0/A:col10/1731111847160/Put/seqid=0 2024-11-09T00:24:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742382_1558 (size=31105) 2024-11-09T00:24:08,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/165d9ece6b4c46a4ac59f57204a650cf 2024-11-09T00:24:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/309b19db4f124f7d9d0f1f6b0681258a is 50, key is test_row_0/B:col10/1731111847160/Put/seqid=0 2024-11-09T00:24:08,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742383_1559 (size=12151) 2024-11-09T00:24:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:08,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111908315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111908315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111908320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111908322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111908425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111908434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,460 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/9517e86c9e194f8aa3dff9317a576cb8 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9517e86c9e194f8aa3dff9317a576cb8 2024-11-09T00:24:08,465 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 9517e86c9e194f8aa3dff9317a576cb8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:08,465 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:08,465 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=11, startTime=1731111848027; duration=0sec 2024-11-09T00:24:08,465 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:08,465 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:08,556 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/309b19db4f124f7d9d0f1f6b0681258a 2024-11-09T00:24:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8490f004db4f4f6ea2d2c134f09d9feb is 50, key is test_row_0/C:col10/1731111847160/Put/seqid=0 2024-11-09T00:24:08,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742384_1560 (size=12151) 2024-11-09T00:24:08,566 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8490f004db4f4f6ea2d2c134f09d9feb 2024-11-09T00:24:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/165d9ece6b4c46a4ac59f57204a650cf as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf 2024-11-09T00:24:08,572 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf, entries=150, sequenceid=156, filesize=30.4 K 2024-11-09T00:24:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/309b19db4f124f7d9d0f1f6b0681258a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a 2024-11-09T00:24:08,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T00:24:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8490f004db4f4f6ea2d2c134f09d9feb as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb 2024-11-09T00:24:08,579 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T00:24:08,580 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c5b55e77595e7e32ac3fc5b9600e8a84 in 459ms, sequenceid=156, compaction requested=false 2024-11-09T00:24:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-09T00:24:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-09T00:24:08,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-09T00:24:08,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-11-09T00:24:08,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.3890 sec 2024-11-09T00:24:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:08,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:08,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T00:24:08,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e0386f1d18324de087776b637b7ca245_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:08,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742385_1561 (size=14794) 2024-11-09T00:24:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111908704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111908705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111908814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:08,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111908814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111909024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111909024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:09,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e0386f1d18324de087776b637b7ca245_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0386f1d18324de087776b637b7ca245_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:09,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/5ec50dc25a4d4a4b9fef49891d75db31, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:09,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/5ec50dc25a4d4a4b9fef49891d75db31 is 175, key is test_row_0/A:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:09,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742386_1562 (size=39749) 2024-11-09T00:24:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111909335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111909335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111909335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111909335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,338 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,339 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T00:24:09,345 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:09,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-09T00:24:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:09,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:09,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:09,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:09,453 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/5ec50dc25a4d4a4b9fef49891d75db31 2024-11-09T00:24:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:09,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/7fa2a3a5451b4265b2264ed447b832e5 is 50, key is test_row_0/B:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742387_1563 (size=12151) 2024-11-09T00:24:09,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:09,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:09,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111909844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111909844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:09,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/7fa2a3a5451b4265b2264ed447b832e5 2024-11-09T00:24:09,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/518bd12a777f421891eb4d7af7309841 is 50, key is test_row_0/C:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:09,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742388_1564 (size=12151) 2024-11-09T00:24:09,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:09,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:10,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:10,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:10,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:10,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/518bd12a777f421891eb4d7af7309841 2024-11-09T00:24:10,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/5ec50dc25a4d4a4b9fef49891d75db31 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31 2024-11-09T00:24:10,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31, entries=200, sequenceid=172, filesize=38.8 K 2024-11-09T00:24:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/7fa2a3a5451b4265b2264ed447b832e5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5 2024-11-09T00:24:10,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T00:24:10,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/518bd12a777f421891eb4d7af7309841 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841 2024-11-09T00:24:10,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T00:24:10,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1650ms, sequenceid=172, compaction requested=true 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:10,287 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:10,287 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:10,288 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:10,288 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:10,288 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:10,288 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,288 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=99.8 K 2024-11-09T00:24:10,288 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,288 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31] 2024-11-09T00:24:10,288 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:10,288 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,288 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9517e86c9e194f8aa3dff9317a576cb8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=35.8 K 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9517e86c9e194f8aa3dff9317a576cb8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846467 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c857b0e574f44be493115442862a2692, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846467 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165d9ece6b4c46a4ac59f57204a650cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731111847150 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 309b19db4f124f7d9d0f1f6b0681258a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731111847150 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa2a3a5451b4265b2264ed447b832e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:10,289 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec50dc25a4d4a4b9fef49891d75db31, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:10,297 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:10,299 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110925c0d2c5f531471297335396d2014a8c_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:10,299 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:10,299 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/43ab21c1d60848b694dd6c1fd36ee653 is 50, key is test_row_0/B:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:10,300 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110925c0d2c5f531471297335396d2014a8c_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:10,300 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110925c0d2c5f531471297335396d2014a8c_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742389_1565 (size=12527) 2024-11-09T00:24:10,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742390_1566 (size=4469) 2024-11-09T00:24:10,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T00:24:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,416 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T00:24:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109108d14a22d6542f59459e0806ecc5056_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111848702/Put/seqid=0 2024-11-09T00:24:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742391_1567 (size=12304) 2024-11-09T00:24:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:10,704 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#476 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:10,704 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/28e911d9c09644c7af3451e70f95f549 is 175, key is test_row_0/A:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:10,707 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/43ab21c1d60848b694dd6c1fd36ee653 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/43ab21c1d60848b694dd6c1fd36ee653 2024-11-09T00:24:10,710 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 43ab21c1d60848b694dd6c1fd36ee653(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:10,710 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:10,711 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=13, startTime=1731111850287; duration=0sec 2024-11-09T00:24:10,711 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:10,711 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:10,711 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:10,711 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:10,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:10,712 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:10,712 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/5f4f0fe56b3d4899b704e616dff23d91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=35.8 K 2024-11-09T00:24:10,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4f0fe56b3d4899b704e616dff23d91, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731111846467 2024-11-09T00:24:10,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8490f004db4f4f6ea2d2c134f09d9feb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731111847150 2024-11-09T00:24:10,712 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 518bd12a777f421891eb4d7af7309841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:10,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742392_1568 (size=31481) 2024-11-09T00:24:10,718 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:10,719 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/957368139b0f41e5ab284dd709256bdd is 50, key is test_row_0/C:col10/1731111848314/Put/seqid=0 2024-11-09T00:24:10,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742393_1569 (size=12527) 2024-11-09T00:24:10,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:10,832 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109108d14a22d6542f59459e0806ecc5056_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109108d14a22d6542f59459e0806ecc5056_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:10,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4d087de5665943f5b9db9a232ceda0f0, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:10,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4d087de5665943f5b9db9a232ceda0f0 is 175, key is test_row_0/A:col10/1731111848702/Put/seqid=0 2024-11-09T00:24:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742394_1570 (size=31105) 2024-11-09T00:24:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:10,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111910878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111910885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111910995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111910995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,121 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/28e911d9c09644c7af3451e70f95f549 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549 2024-11-09T00:24:11,125 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into 28e911d9c09644c7af3451e70f95f549(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:11,125 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:11,125 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=13, startTime=1731111850287; duration=0sec 2024-11-09T00:24:11,125 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:11,126 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:11,145 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/957368139b0f41e5ab284dd709256bdd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/957368139b0f41e5ab284dd709256bdd 2024-11-09T00:24:11,150 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 957368139b0f41e5ab284dd709256bdd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:11,150 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:11,150 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=13, startTime=1731111850287; duration=0sec 2024-11-09T00:24:11,150 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:11,150 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111911204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111911205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,237 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4d087de5665943f5b9db9a232ceda0f0 2024-11-09T00:24:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/95f4868320a4490091ab52b865d8d0e2 is 50, key is test_row_0/B:col10/1731111848702/Put/seqid=0 2024-11-09T00:24:11,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742395_1571 (size=12151) 2024-11-09T00:24:11,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111911354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111911354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,357 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:11,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,357 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:11,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111911444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,446 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111911515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111911515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:11,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/95f4868320a4490091ab52b865d8d0e2 2024-11-09T00:24:11,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af17c41e88d04eb6899b34cc3a59f6b3 is 50, key is test_row_0/C:col10/1731111848702/Put/seqid=0 2024-11-09T00:24:11,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742396_1572 (size=12151) 2024-11-09T00:24:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111912024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111912024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:12,074 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af17c41e88d04eb6899b34cc3a59f6b3 2024-11-09T00:24:12,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/4d087de5665943f5b9db9a232ceda0f0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0 2024-11-09T00:24:12,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0, entries=150, sequenceid=194, filesize=30.4 K 2024-11-09T00:24:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/95f4868320a4490091ab52b865d8d0e2 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2 2024-11-09T00:24:12,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-09T00:24:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af17c41e88d04eb6899b34cc3a59f6b3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3 2024-11-09T00:24:12,098 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3, entries=150, sequenceid=194, filesize=11.9 K 2024-11-09T00:24:12,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1683ms, sequenceid=194, compaction requested=false 2024-11-09T00:24:12,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:12,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:12,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-09T00:24:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-09T00:24:12,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T00:24:12,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7530 sec 2024-11-09T00:24:12,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7570 sec 2024-11-09T00:24:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:13,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T00:24:13,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b64cbf794dbe4420964f20867a109e87_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:13,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742397_1573 (size=14794) 2024-11-09T00:24:13,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:13,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b64cbf794dbe4420964f20867a109e87_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b64cbf794dbe4420964f20867a109e87_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:13,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0b75c145b4b34ee6a45c0f4fd21b2718, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:13,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0b75c145b4b34ee6a45c0f4fd21b2718 is 175, key is test_row_0/A:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:13,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742398_1574 (size=39749) 2024-11-09T00:24:13,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0b75c145b4b34ee6a45c0f4fd21b2718 2024-11-09T00:24:13,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1d097090014a409c9b5dcc2fbcb6aa1e is 50, key is test_row_0/B:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:13,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742399_1575 (size=12151) 2024-11-09T00:24:13,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111913077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111913081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111913184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111913195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111913394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111913405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1d097090014a409c9b5dcc2fbcb6aa1e 2024-11-09T00:24:13,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/9615e53747e5458f82b2615e2ec780ac is 50, key is test_row_0/C:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:13,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742400_1576 (size=12151) 2024-11-09T00:24:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T00:24:13,514 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:13,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-09T00:24:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T00:24:13,516 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:13,516 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:13,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T00:24:13,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T00:24:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111913704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111913715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:13,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T00:24:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T00:24:13,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/9615e53747e5458f82b2615e2ec780ac 2024-11-09T00:24:13,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0b75c145b4b34ee6a45c0f4fd21b2718 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718 2024-11-09T00:24:13,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718, entries=200, sequenceid=213, filesize=38.8 K 2024-11-09T00:24:13,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1d097090014a409c9b5dcc2fbcb6aa1e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e 2024-11-09T00:24:13,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T00:24:13,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/9615e53747e5458f82b2615e2ec780ac as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac 2024-11-09T00:24:13,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T00:24:13,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c5b55e77595e7e32ac3fc5b9600e8a84 in 857ms, sequenceid=213, compaction requested=true 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:13,895 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:13,895 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:13,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:13,896 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:13,896 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:13,896 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:13,896 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:13,896 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,896 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,896 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/43ab21c1d60848b694dd6c1fd36ee653, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.0 K 2024-11-09T00:24:13,896 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=99.9 K 2024-11-09T00:24:13,896 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,896 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718] 2024-11-09T00:24:13,899 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ab21c1d60848b694dd6c1fd36ee653, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:13,899 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e911d9c09644c7af3451e70f95f549, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:13,899 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d087de5665943f5b9db9a232ceda0f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111848687 2024-11-09T00:24:13,899 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f4868320a4490091ab52b865d8d0e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111848687 2024-11-09T00:24:13,900 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b75c145b4b34ee6a45c0f4fd21b2718, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:13,900 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d097090014a409c9b5dcc2fbcb6aa1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:13,907 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:13,908 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/8b476faff4e04b1e8646cbef449f6261 is 50, key is test_row_0/B:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:13,908 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:13,910 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411093667f80d74544c17b6897109d1991bc6_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:13,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742401_1577 (size=12629) 2024-11-09T00:24:13,912 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411093667f80d74544c17b6897109d1991bc6_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:13,912 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093667f80d74544c17b6897109d1991bc6_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:13,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742402_1578 (size=4469) 2024-11-09T00:24:13,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T00:24:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:13,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T00:24:13,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cf3db9617b1f4453b6deae02a35de603_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111853076/Put/seqid=0 2024-11-09T00:24:13,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742403_1579 (size=12304) 2024-11-09T00:24:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T00:24:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:14,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:14,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111914249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111914250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,315 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/8b476faff4e04b1e8646cbef449f6261 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/8b476faff4e04b1e8646cbef449f6261 2024-11-09T00:24:14,319 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:14,319 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/fa73734518904fc68d42ce3539ca0a4f is 175, key is test_row_0/A:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:14,321 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 8b476faff4e04b1e8646cbef449f6261(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:14,321 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:14,321 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=13, startTime=1731111853895; duration=0sec 2024-11-09T00:24:14,321 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:14,321 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:14,321 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:14,322 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:14,322 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:14,322 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,322 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/957368139b0f41e5ab284dd709256bdd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.0 K 2024-11-09T00:24:14,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 957368139b0f41e5ab284dd709256bdd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731111848313 2024-11-09T00:24:14,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting af17c41e88d04eb6899b34cc3a59f6b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731111848687 2024-11-09T00:24:14,323 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9615e53747e5458f82b2615e2ec780ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:14,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742404_1580 (size=31583) 2024-11-09T00:24:14,328 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:14,328 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/23d4bba4296142a7a89ce9bc002240dd is 50, key is test_row_0/C:col10/1731111850877/Put/seqid=0 2024-11-09T00:24:14,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742405_1581 (size=12629) 2024-11-09T00:24:14,334 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/23d4bba4296142a7a89ce9bc002240dd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/23d4bba4296142a7a89ce9bc002240dd 2024-11-09T00:24:14,337 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 23d4bba4296142a7a89ce9bc002240dd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:14,337 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:14,337 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=13, startTime=1731111853895; duration=0sec 2024-11-09T00:24:14,337 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:14,337 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111914365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111914365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:14,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cf3db9617b1f4453b6deae02a35de603_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf3db9617b1f4453b6deae02a35de603_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0c54d05138a1486d9ea1a23bef5a8c82, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0c54d05138a1486d9ea1a23bef5a8c82 is 175, key is test_row_0/A:col10/1731111853076/Put/seqid=0 2024-11-09T00:24:14,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742406_1582 (size=31105) 2024-11-09T00:24:14,395 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0c54d05138a1486d9ea1a23bef5a8c82 2024-11-09T00:24:14,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/450a847a3fb44c55a015d7e52de2f76c is 50, key is test_row_0/B:col10/1731111853076/Put/seqid=0 2024-11-09T00:24:14,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742407_1583 (size=12151) 2024-11-09T00:24:14,404 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/450a847a3fb44c55a015d7e52de2f76c 2024-11-09T00:24:14,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/89194ed016624eb9af5e1f862d770248 is 50, key is test_row_0/C:col10/1731111853076/Put/seqid=0 2024-11-09T00:24:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742408_1584 (size=12151) 2024-11-09T00:24:14,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/89194ed016624eb9af5e1f862d770248 2024-11-09T00:24:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/0c54d05138a1486d9ea1a23bef5a8c82 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82 2024-11-09T00:24:14,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82, entries=150, sequenceid=233, filesize=30.4 K 2024-11-09T00:24:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/450a847a3fb44c55a015d7e52de2f76c as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c 2024-11-09T00:24:14,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T00:24:14,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/89194ed016624eb9af5e1f862d770248 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248 2024-11-09T00:24:14,426 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248, entries=150, sequenceid=233, filesize=11.9 K 2024-11-09T00:24:14,427 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c5b55e77595e7e32ac3fc5b9600e8a84 in 453ms, sequenceid=233, compaction requested=false 2024-11-09T00:24:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-09T00:24:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-09T00:24:14,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-09T00:24:14,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 912 msec 2024-11-09T00:24:14,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 915 msec 2024-11-09T00:24:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:14,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T00:24:14,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110994ff71b2606045a1a05582e07d52885f_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:14,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742409_1585 (size=12304) 2024-11-09T00:24:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111914615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111914621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T00:24:14,654 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:14,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-09T00:24:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:14,656 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:14,657 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:14,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:14,728 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/fa73734518904fc68d42ce3539ca0a4f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f 2024-11-09T00:24:14,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111914725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,731 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into fa73734518904fc68d42ce3539ca0a4f(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:14,732 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:14,732 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=13, startTime=1731111853895; duration=0sec 2024-11-09T00:24:14,732 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:14,732 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111914734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:14,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111914934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111914944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:14,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:14,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:14,988 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:14,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110994ff71b2606045a1a05582e07d52885f_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110994ff71b2606045a1a05582e07d52885f_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:14,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/ad96964fd47449ceb96c1a765dd51037, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:14,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/ad96964fd47449ceb96c1a765dd51037 is 175, key is test_row_0/A:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742410_1586 (size=31105) 2024-11-09T00:24:15,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111915245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111915254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33368 deadline: 1731111915384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33332 deadline: 1731111915384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,387 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_0' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:15,387 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/ad96964fd47449ceb96c1a765dd51037 2024-11-09T00:24:15,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/4aee329715ca4e8eac5c52b8580f0218 is 50, key is test_row_0/B:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742411_1587 (size=12151) 2024-11-09T00:24:15,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/4aee329715ca4e8eac5c52b8580f0218 2024-11-09T00:24:15,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8ace05ede7c54a678aaf9d8d2fdb4281 is 50, key is test_row_0/C:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:15,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742412_1588 (size=12151) 2024-11-09T00:24:15,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8ace05ede7c54a678aaf9d8d2fdb4281 2024-11-09T00:24:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/ad96964fd47449ceb96c1a765dd51037 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037 2024-11-09T00:24:15,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037, entries=150, sequenceid=252, filesize=30.4 K 2024-11-09T00:24:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:15,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/4aee329715ca4e8eac5c52b8580f0218 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218 2024-11-09T00:24:15,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T00:24:15,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/8ace05ede7c54a678aaf9d8d2fdb4281 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281 2024-11-09T00:24:15,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T00:24:15,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c5b55e77595e7e32ac3fc5b9600e8a84 in 851ms, sequenceid=252, compaction requested=true 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:15,428 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:15,428 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:15,429 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:15,429 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:15,429 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,429 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:15,429 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=91.6 K 2024-11-09T00:24:15,429 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:15,429 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,429 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/8b476faff4e04b1e8646cbef449f6261, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.1 K 2024-11-09T00:24:15,429 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,429 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037] 2024-11-09T00:24:15,430 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b476faff4e04b1e8646cbef449f6261, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:15,430 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa73734518904fc68d42ce3539ca0a4f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:15,430 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 450a847a3fb44c55a015d7e52de2f76c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731111853075 2024-11-09T00:24:15,430 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c54d05138a1486d9ea1a23bef5a8c82, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731111853075 2024-11-09T00:24:15,431 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aee329715ca4e8eac5c52b8580f0218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:15,432 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad96964fd47449ceb96c1a765dd51037, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:15,438 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:15,438 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/9f0437bf927643bfa844b2eaaa303809 is 50, key is test_row_0/B:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:15,439 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:15,453 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109187ba3a0c21e48fdaa9b42d7ece5498d_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:15,455 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109187ba3a0c21e48fdaa9b42d7ece5498d_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:15,455 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109187ba3a0c21e48fdaa9b42d7ece5498d_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:15,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742413_1589 (size=12731) 2024-11-09T00:24:15,471 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/9f0437bf927643bfa844b2eaaa303809 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9f0437bf927643bfa844b2eaaa303809 2024-11-09T00:24:15,476 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 9f0437bf927643bfa844b2eaaa303809(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:15,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:15,476 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=13, startTime=1731111855428; duration=0sec 2024-11-09T00:24:15,476 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:15,477 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:15,477 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:15,477 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:15,477 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:15,478 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,478 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/23d4bba4296142a7a89ce9bc002240dd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.1 K 2024-11-09T00:24:15,478 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d4bba4296142a7a89ce9bc002240dd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731111850877 2024-11-09T00:24:15,478 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 89194ed016624eb9af5e1f862d770248, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731111853075 2024-11-09T00:24:15,478 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ace05ede7c54a678aaf9d8d2fdb4281, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:15,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742414_1590 (size=4469) 2024-11-09T00:24:15,480 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#495 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:15,481 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/461745e2bae04825888848c9b9ab7b81 is 175, key is test_row_0/A:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742415_1591 (size=31685) 2024-11-09T00:24:15,485 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:15,485 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/27b69f9edf0d448b928bdbbaf62f0919 is 50, key is test_row_0/C:col10/1731111854248/Put/seqid=0 2024-11-09T00:24:15,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742416_1592 (size=12731) 2024-11-09T00:24:15,494 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/27b69f9edf0d448b928bdbbaf62f0919 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/27b69f9edf0d448b928bdbbaf62f0919 2024-11-09T00:24:15,500 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 27b69f9edf0d448b928bdbbaf62f0919(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:15,500 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=13, startTime=1731111855428; duration=0sec 2024-11-09T00:24:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:15,500 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:15,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T00:24:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:15,572 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T00:24:15,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976b2cb88192e4d84b35d44768e1390b7_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111854614/Put/seqid=0 2024-11-09T00:24:15,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742417_1593 (size=12454) 2024-11-09T00:24:15,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:15,590 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976b2cb88192e4d84b35d44768e1390b7_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976b2cb88192e4d84b35d44768e1390b7_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d48bdd528d5d48668283d4f5b2b7fc92, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d48bdd528d5d48668283d4f5b2b7fc92 is 175, key is test_row_0/A:col10/1731111854614/Put/seqid=0 2024-11-09T00:24:15,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742418_1594 (size=31255) 2024-11-09T00:24:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:15,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111915787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:15,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111915791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,888 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/461745e2bae04825888848c9b9ab7b81 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81 2024-11-09T00:24:15,891 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into 461745e2bae04825888848c9b9ab7b81(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:15,891 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:15,891 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=13, startTime=1731111855428; duration=0sec 2024-11-09T00:24:15,891 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:15,891 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111915894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111915904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:15,995 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d48bdd528d5d48668283d4f5b2b7fc92 2024-11-09T00:24:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1c5008b7523849e1a8600ccd4eaafc5b is 50, key is test_row_0/B:col10/1731111854614/Put/seqid=0 2024-11-09T00:24:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742419_1595 (size=12301) 2024-11-09T00:24:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111916105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111916115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:16,405 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1c5008b7523849e1a8600ccd4eaafc5b 2024-11-09T00:24:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af995a0068ff4ed8aeef1e5a83b042d6 is 50, key is test_row_0/C:col10/1731111854614/Put/seqid=0 2024-11-09T00:24:16,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742420_1596 (size=12301) 2024-11-09T00:24:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111916414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111916424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:16,814 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af995a0068ff4ed8aeef1e5a83b042d6 2024-11-09T00:24:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d48bdd528d5d48668283d4f5b2b7fc92 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92 2024-11-09T00:24:16,819 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92, entries=150, sequenceid=274, filesize=30.5 K 2024-11-09T00:24:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/1c5008b7523849e1a8600ccd4eaafc5b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b 2024-11-09T00:24:16,822 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T00:24:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/af995a0068ff4ed8aeef1e5a83b042d6 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6 2024-11-09T00:24:16,825 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T00:24:16,825 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1253ms, sequenceid=274, compaction requested=false 2024-11-09T00:24:16,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:16,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:16,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-09T00:24:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-09T00:24:16,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T00:24:16,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-09T00:24:16,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1850 sec 2024-11-09T00:24:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:16,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:16,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T00:24:16,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094350454cd2454f9b8dc3268267dc0986_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742421_1597 (size=14994) 2024-11-09T00:24:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111916961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111916965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111917074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111917074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111917284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111917284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:17,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094350454cd2454f9b8dc3268267dc0986_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094350454cd2454f9b8dc3268267dc0986_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:17,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/552e2a5a29d34b5086d92c0c61c2d394, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:17,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/552e2a5a29d34b5086d92c0c61c2d394 is 175, key is test_row_0/A:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742422_1598 (size=39949) 2024-11-09T00:24:17,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/552e2a5a29d34b5086d92c0c61c2d394 2024-11-09T00:24:17,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/d26b84d79605452990c45a5ea20c1984 is 50, key is test_row_0/B:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:17,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742423_1599 (size=12301) 2024-11-09T00:24:17,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/d26b84d79605452990c45a5ea20c1984 2024-11-09T00:24:17,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/a5c1155eed2a4d5ab8c9d99998f63e6b is 50, key is test_row_0/C:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742424_1600 (size=12301) 2024-11-09T00:24:17,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111917594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111917595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:17,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/a5c1155eed2a4d5ab8c9d99998f63e6b 2024-11-09T00:24:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/552e2a5a29d34b5086d92c0c61c2d394 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394 2024-11-09T00:24:17,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394, entries=200, sequenceid=293, filesize=39.0 K 2024-11-09T00:24:17,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/d26b84d79605452990c45a5ea20c1984 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984 2024-11-09T00:24:17,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T00:24:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/a5c1155eed2a4d5ab8c9d99998f63e6b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b 2024-11-09T00:24:17,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T00:24:17,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c5b55e77595e7e32ac3fc5b9600e8a84 in 849ms, sequenceid=293, compaction requested=true 2024-11-09T00:24:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:17,776 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:17,776 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:17,776 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=100.5 K 2024-11-09T00:24:17,776 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9f0437bf927643bfa844b2eaaa303809, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.5 K 2024-11-09T00:24:17,776 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:17,776 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394] 2024-11-09T00:24:17,777 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0437bf927643bfa844b2eaaa303809, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:17,777 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461745e2bae04825888848c9b9ab7b81, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:17,777 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c5008b7523849e1a8600ccd4eaafc5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731111854609 2024-11-09T00:24:17,778 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48bdd528d5d48668283d4f5b2b7fc92, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731111854609 2024-11-09T00:24:17,778 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting d26b84d79605452990c45a5ea20c1984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:17,778 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552e2a5a29d34b5086d92c0c61c2d394, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:17,783 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:17,783 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/77f51f8df91f4ca29f1106e1b1ef5c25 is 50, key is test_row_0/B:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:17,784 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:17,788 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d675f0b31ecf405fb858b484f96a7bda_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:17,790 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d675f0b31ecf405fb858b484f96a7bda_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:17,790 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d675f0b31ecf405fb858b484f96a7bda_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742425_1601 (size=12983) 2024-11-09T00:24:17,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742426_1602 (size=4469) 2024-11-09T00:24:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:18,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:18,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T00:24:18,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d8e17a675e064826a29d728c24a852f0_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111856954/Put/seqid=0 2024-11-09T00:24:18,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742427_1603 (size=17534) 2024-11-09T00:24:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111918136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111918138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,201 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/77f51f8df91f4ca29f1106e1b1ef5c25 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/77f51f8df91f4ca29f1106e1b1ef5c25 2024-11-09T00:24:18,202 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:18,202 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a0652c92094240e7b497d644d078e1b5 is 175, key is test_row_0/A:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742428_1604 (size=31937) 2024-11-09T00:24:18,205 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 77f51f8df91f4ca29f1106e1b1ef5c25(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:18,205 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:18,205 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=13, startTime=1731111857776; duration=0sec 2024-11-09T00:24:18,205 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:18,205 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:18,205 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T00:24:18,206 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T00:24:18,206 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:18,206 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:18,206 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/27b69f9edf0d448b928bdbbaf62f0919, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=36.5 K 2024-11-09T00:24:18,206 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b69f9edf0d448b928bdbbaf62f0919, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731111854247 2024-11-09T00:24:18,206 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting af995a0068ff4ed8aeef1e5a83b042d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731111854609 2024-11-09T00:24:18,207 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c1155eed2a4d5ab8c9d99998f63e6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:18,211 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:18,212 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/fc32b3bb0e60451eb484e3ce5863c14d is 50, key is test_row_0/C:col10/1731111855785/Put/seqid=0 2024-11-09T00:24:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742429_1605 (size=12983) 2024-11-09T00:24:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111918244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111918244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111918454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111918454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:18,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d8e17a675e064826a29d728c24a852f0_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d8e17a675e064826a29d728c24a852f0_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:18,529 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c7173ea67d1743159145c8d7f7bf552f, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:18,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c7173ea67d1743159145c8d7f7bf552f is 175, key is test_row_0/A:col10/1731111856954/Put/seqid=0 2024-11-09T00:24:18,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742430_1606 (size=48639) 2024-11-09T00:24:18,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c7173ea67d1743159145c8d7f7bf552f 2024-11-09T00:24:18,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0be5acc558474c2c8a143e3d67d3d817 is 50, key is test_row_0/B:col10/1731111856954/Put/seqid=0 2024-11-09T00:24:18,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742431_1607 (size=12301) 2024-11-09T00:24:18,608 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a0652c92094240e7b497d644d078e1b5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5 2024-11-09T00:24:18,612 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into a0652c92094240e7b497d644d078e1b5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:18,612 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:18,612 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=13, startTime=1731111857775; duration=0sec 2024-11-09T00:24:18,612 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:18,612 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:18,618 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/fc32b3bb0e60451eb484e3ce5863c14d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/fc32b3bb0e60451eb484e3ce5863c14d 2024-11-09T00:24:18,621 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into fc32b3bb0e60451eb484e3ce5863c14d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:18,621 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:18,621 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=13, startTime=1731111857776; duration=0sec 2024-11-09T00:24:18,621 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:18,621 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111918764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111918764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T00:24:18,814 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:18,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T00:24:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-09T00:24:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:18,816 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T00:24:18,816 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T00:24:18,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T00:24:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:18,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0be5acc558474c2c8a143e3d67d3d817 2024-11-09T00:24:18,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/90e03d0e92694642b2ea67feced57844 is 50, key is test_row_0/C:col10/1731111856954/Put/seqid=0 2024-11-09T00:24:18,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T00:24:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:18,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T00:24:18,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742432_1608 (size=12301) 2024-11-09T00:24:18,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/90e03d0e92694642b2ea67feced57844 2024-11-09T00:24:18,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/c7173ea67d1743159145c8d7f7bf552f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f 2024-11-09T00:24:18,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f, entries=250, sequenceid=313, filesize=47.5 K 2024-11-09T00:24:18,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0be5acc558474c2c8a143e3d67d3d817 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817 2024-11-09T00:24:18,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T00:24:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/90e03d0e92694642b2ea67feced57844 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844 2024-11-09T00:24:18,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844, entries=150, sequenceid=313, filesize=12.0 K 2024-11-09T00:24:18,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c5b55e77595e7e32ac3fc5b9600e8a84 in 876ms, sequenceid=313, compaction requested=false 2024-11-09T00:24:18,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:19,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35861 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T00:24:19,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:19,122 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:24:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aafb92c48b784613946db0ff2fb36dd9_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111858111/Put/seqid=0 2024-11-09T00:24:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742433_1609 (size=12454) 2024-11-09T00:24:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:19,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. as already flushing 2024-11-09T00:24:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111919316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111919319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111919424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111919424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:19,534 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109aafb92c48b784613946db0ff2fb36dd9_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aafb92c48b784613946db0ff2fb36dd9_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/61e34adc2da74a1abeed78685a3d13d3, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/61e34adc2da74a1abeed78685a3d13d3 is 175, key is test_row_0/A:col10/1731111858111/Put/seqid=0 2024-11-09T00:24:19,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742434_1610 (size=31255) 2024-11-09T00:24:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111919634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111919634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111919945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/61e34adc2da74a1abeed78685a3d13d3 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111919945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/407b4cb722934ba4b551d0a195814a8e is 50, key is test_row_0/B:col10/1731111858111/Put/seqid=0 2024-11-09T00:24:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:19,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742435_1611 (size=12301) 2024-11-09T00:24:19,955 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/407b4cb722934ba4b551d0a195814a8e 2024-11-09T00:24:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/428651884e354950839ddfd5c33022bd is 50, key is test_row_0/C:col10/1731111858111/Put/seqid=0 2024-11-09T00:24:19,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742436_1612 (size=12301) 2024-11-09T00:24:20,371 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/428651884e354950839ddfd5c33022bd 2024-11-09T00:24:20,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/61e34adc2da74a1abeed78685a3d13d3 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3 2024-11-09T00:24:20,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3, entries=150, sequenceid=332, filesize=30.5 K 2024-11-09T00:24:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/407b4cb722934ba4b551d0a195814a8e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e 2024-11-09T00:24:20,380 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T00:24:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/428651884e354950839ddfd5c33022bd as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd 2024-11-09T00:24:20,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T00:24:20,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1262ms, sequenceid=332, compaction requested=true 2024-11-09T00:24:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef986b38239e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-09T00:24:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-09T00:24:20,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-09T00:24:20,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-09T00:24:20,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5730 sec 2024-11-09T00:24:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:20,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:20,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T00:24:20,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ed54d3252fb249feb1efb4483c1831e5_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:20,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742437_1613 (size=14994) 2024-11-09T00:24:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111920491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111920492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111920604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111920606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,749 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-09T00:24:20,749 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:20,749 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,749 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:20,749 INFO [Thread-2335 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2335. 2024-11-09T00:24:20,749 DEBUG [Thread-2335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:20,749 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,749 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:20,751 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-11-09T00:24:20,751 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:20,751 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,751 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:20,752 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-09T00:24:20,752 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:20,752 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,753 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:20,753 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-11-09T00:24:20,753 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:20,753 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,753 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:20,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111920814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111920814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:20,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:20,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ed54d3252fb249feb1efb4483c1831e5_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed54d3252fb249feb1efb4483c1831e5_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:20,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d0b72207d197419daf8de2e815d47a43, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:20,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d0b72207d197419daf8de2e815d47a43 is 175, key is test_row_0/A:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:20,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742438_1614 (size=39949) 2024-11-09T00:24:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T00:24:20,965 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111921124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111921124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:21,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d0b72207d197419daf8de2e815d47a43 2024-11-09T00:24:21,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/87aed9da32d24aa3ad0e9526f08b1be0 is 50, key is test_row_0/B:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:21,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742439_1615 (size=12301) 2024-11-09T00:24:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33392 deadline: 1731111921515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:21,517 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef986b38239e,35861,1731111692607 for 'test_row_1' in c5b55e77595e7e32ac3fc5b9600e8a84 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T00:24:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T00:24:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1731111921635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35861 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33348 deadline: 1731111921635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 , the old value is region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., hostname=ef986b38239e,35861,1731111692607, seqNum=5 because the exception is null or not the one we care about 2024-11-09T00:24:21,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/87aed9da32d24aa3ad0e9526f08b1be0 2024-11-09T00:24:21,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/64ff87f269664a4f8cc7189126410dc9 is 50, key is test_row_0/C:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:21,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742440_1616 (size=12301) 2024-11-09T00:24:22,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/64ff87f269664a4f8cc7189126410dc9 2024-11-09T00:24:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/d0b72207d197419daf8de2e815d47a43 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43 2024-11-09T00:24:22,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43, entries=200, sequenceid=354, filesize=39.0 K 2024-11-09T00:24:22,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/87aed9da32d24aa3ad0e9526f08b1be0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0 2024-11-09T00:24:22,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0, entries=150, sequenceid=354, filesize=12.0 K 2024-11-09T00:24:22,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/64ff87f269664a4f8cc7189126410dc9 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9 2024-11-09T00:24:22,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9, entries=150, sequenceid=354, filesize=12.0 K 2024-11-09T00:24:22,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1670ms, sequenceid=354, compaction requested=true 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T00:24:22,129 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c5b55e77595e7e32ac3fc5b9600e8a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:22,129 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/B is initiating minor compaction (all files) 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/A is initiating minor compaction (all files) 2024-11-09T00:24:22,130 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/B in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:22,130 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/A in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:22,130 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/77f51f8df91f4ca29f1106e1b1ef5c25, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=48.7 K 2024-11-09T00:24:22,130 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=148.2 K 2024-11-09T00:24:22,130 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. files: [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43] 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f51f8df91f4ca29f1106e1b1ef5c25, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0652c92094240e7b497d644d078e1b5, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:22,130 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be5acc558474c2c8a143e3d67d3d817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731111856954 2024-11-09T00:24:22,131 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7173ea67d1743159145c8d7f7bf552f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731111856954 2024-11-09T00:24:22,131 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 407b4cb722934ba4b551d0a195814a8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111858111 2024-11-09T00:24:22,131 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e34adc2da74a1abeed78685a3d13d3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111858111 2024-11-09T00:24:22,131 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 87aed9da32d24aa3ad0e9526f08b1be0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731111859313 2024-11-09T00:24:22,131 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b72207d197419daf8de2e815d47a43, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731111859303 2024-11-09T00:24:22,136 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:22,137 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#B#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:22,137 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109248f682d89e44f99b78d631ffe0ccfd4_c5b55e77595e7e32ac3fc5b9600e8a84 store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:22,137 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/2489e4edb31d41529e40a3f3f04fe3d7 is 50, key is test_row_0/B:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:22,141 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109248f682d89e44f99b78d631ffe0ccfd4_c5b55e77595e7e32ac3fc5b9600e8a84, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:22,141 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109248f682d89e44f99b78d631ffe0ccfd4_c5b55e77595e7e32ac3fc5b9600e8a84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742441_1617 (size=13119) 2024-11-09T00:24:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742442_1618 (size=4469) 2024-11-09T00:24:22,549 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/2489e4edb31d41529e40a3f3f04fe3d7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/2489e4edb31d41529e40a3f3f04fe3d7 2024-11-09T00:24:22,550 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:22,550 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/8f553b4fae2e4c6e97582323da4dec1e is 175, key is test_row_0/A:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:22,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742443_1619 (size=32073) 2024-11-09T00:24:22,556 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/B of c5b55e77595e7e32ac3fc5b9600e8a84 into 2489e4edb31d41529e40a3f3f04fe3d7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:22,557 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:22,557 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/B, priority=12, startTime=1731111862129; duration=0sec 2024-11-09T00:24:22,557 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T00:24:22,557 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:B 2024-11-09T00:24:22,557 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T00:24:22,559 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T00:24:22,559 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1541): c5b55e77595e7e32ac3fc5b9600e8a84/C is initiating minor compaction (all files) 2024-11-09T00:24:22,559 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c5b55e77595e7e32ac3fc5b9600e8a84/C in TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:22,559 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/fc32b3bb0e60451eb484e3ce5863c14d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9] into tmpdir=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp, totalSize=48.7 K 2024-11-09T00:24:22,559 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting fc32b3bb0e60451eb484e3ce5863c14d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731111855785 2024-11-09T00:24:22,560 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e03d0e92694642b2ea67feced57844, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731111856954 2024-11-09T00:24:22,560 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 428651884e354950839ddfd5c33022bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731111858111 2024-11-09T00:24:22,561 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ff87f269664a4f8cc7189126410dc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731111859313 2024-11-09T00:24:22,569 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c5b55e77595e7e32ac3fc5b9600e8a84#C#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T00:24:22,569 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/0bc96fa37538448a95383a0e9a936ffc is 50, key is test_row_0/C:col10/1731111860458/Put/seqid=0 2024-11-09T00:24:22,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742444_1620 (size=13119) 2024-11-09T00:24:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35861 {}] regionserver.HRegion(8855): Flush requested on c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T00:24:22,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T00:24:22,646 INFO [Thread-2322 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2322. 2024-11-09T00:24:22,646 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-09T00:24:22,646 DEBUG [Thread-2322 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:22,646 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:22,646 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:22,646 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:22,646 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:22,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:22,646 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:22,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:22,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f2d287ecd65a4d80b372f519ffe2c727_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111862644/Put/seqid=0 2024-11-09T00:24:22,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742445_1621 (size=12454) 2024-11-09T00:24:22,961 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/8f553b4fae2e4c6e97582323da4dec1e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8f553b4fae2e4c6e97582323da4dec1e 2024-11-09T00:24:22,966 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/A of c5b55e77595e7e32ac3fc5b9600e8a84 into 8f553b4fae2e4c6e97582323da4dec1e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:22,966 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:22,966 INFO [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/A, priority=12, startTime=1731111862129; duration=0sec 2024-11-09T00:24:22,966 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:22,966 DEBUG [RS:0;ef986b38239e:35861-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:A 2024-11-09T00:24:22,979 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/0bc96fa37538448a95383a0e9a936ffc as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/0bc96fa37538448a95383a0e9a936ffc 2024-11-09T00:24:22,985 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c5b55e77595e7e32ac3fc5b9600e8a84/C of c5b55e77595e7e32ac3fc5b9600e8a84 into 0bc96fa37538448a95383a0e9a936ffc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T00:24:22,985 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:22,985 INFO [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., storeName=c5b55e77595e7e32ac3fc5b9600e8a84/C, priority=12, startTime=1731111862129; duration=0sec 2024-11-09T00:24:22,985 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T00:24:22,985 DEBUG [RS:0;ef986b38239e:35861-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c5b55e77595e7e32ac3fc5b9600e8a84:C 2024-11-09T00:24:23,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:23,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f2d287ecd65a4d80b372f519ffe2c727_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f2d287ecd65a4d80b372f519ffe2c727_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:23,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/725987721a9b400883766b735630f4f7, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:23,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/725987721a9b400883766b735630f4f7 is 175, key is test_row_0/A:col10/1731111862644/Put/seqid=0 2024-11-09T00:24:23,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742446_1622 (size=31255) 2024-11-09T00:24:23,467 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/725987721a9b400883766b735630f4f7 2024-11-09T00:24:23,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0db080e895da431d86cb44b8ff9d364a is 50, key is test_row_0/B:col10/1731111862644/Put/seqid=0 2024-11-09T00:24:23,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742447_1623 (size=12301) 2024-11-09T00:24:23,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0db080e895da431d86cb44b8ff9d364a 2024-11-09T00:24:23,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/bee9c75497e747ee9537212ebaaa844a is 50, key is test_row_0/C:col10/1731111862644/Put/seqid=0 2024-11-09T00:24:23,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742448_1624 (size=12301) 2024-11-09T00:24:24,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/bee9c75497e747ee9537212ebaaa844a 2024-11-09T00:24:24,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/725987721a9b400883766b735630f4f7 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/725987721a9b400883766b735630f4f7 2024-11-09T00:24:24,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/725987721a9b400883766b735630f4f7, entries=150, sequenceid=371, filesize=30.5 K 2024-11-09T00:24:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/0db080e895da431d86cb44b8ff9d364a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0db080e895da431d86cb44b8ff9d364a 2024-11-09T00:24:24,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0db080e895da431d86cb44b8ff9d364a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-09T00:24:24,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/bee9c75497e747ee9537212ebaaa844a as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/bee9c75497e747ee9537212ebaaa844a 2024-11-09T00:24:24,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/bee9c75497e747ee9537212ebaaa844a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-09T00:24:24,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1664ms, sequenceid=371, compaction requested=false 2024-11-09T00:24:24,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c5b55e77595e7e32ac3fc5b9600e8a84: 2024-11-09T00:24:25,396 INFO [Thread-2324 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2324. 2024-11-09T00:24:25,396 DEBUG [Thread-2324 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:25,396 DEBUG [Thread-2324 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:25,396 DEBUG [Thread-2324 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:25,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:25,406 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-09T00:24:25,406 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:25,406 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:25,406 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:25,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:30,119 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T00:24:31,560 INFO [Thread-2326 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2326. 2024-11-09T00:24:31,560 DEBUG [Thread-2326 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T00:24:31,560 DEBUG [Thread-2326 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:31,560 DEBUG [Thread-2326 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:31,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3514 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3592 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3419 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3506 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3562 2024-11-09T00:24:31,561 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T00:24:31,562 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:24:31,562 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:24:31,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:31,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:31,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:31,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T00:24:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:31,567 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111871567"}]},"ts":"1731111871567"} 2024-11-09T00:24:31,569 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T00:24:31,569 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T00:24:31,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T00:24:31,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, UNASSIGN}] 2024-11-09T00:24:31,573 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, UNASSIGN 2024-11-09T00:24:31,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=CLOSING, regionLocation=ef986b38239e,35861,1731111692607 2024-11-09T00:24:31,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, UNASSIGN because future has completed 2024-11-09T00:24:31,575 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T00:24:31,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607}] 2024-11-09T00:24:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:31,729 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:31,729 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T00:24:31,730 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing c5b55e77595e7e32ac3fc5b9600e8a84, disabling compactions & flushes 2024-11-09T00:24:31,730 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:31,730 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:31,730 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. after waiting 0 ms 2024-11-09T00:24:31,730 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:31,730 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing c5b55e77595e7e32ac3fc5b9600e8a84 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-09T00:24:31,739 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109310e6b2aa9b84654b61e5bbb45ecd1a6_c5b55e77595e7e32ac3fc5b9600e8a84 is 50, key is test_row_0/A:col10/1731111865395/Put/seqid=0 2024-11-09T00:24:31,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742449_1625 (size=9914) 2024-11-09T00:24:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:32,143 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T00:24:32,149 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109310e6b2aa9b84654b61e5bbb45ecd1a6_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109310e6b2aa9b84654b61e5bbb45ecd1a6_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:32,150 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a9b9478dd5b14016bc0c728a79b2b64f, store: [table=TestAcidGuarantees family=A region=c5b55e77595e7e32ac3fc5b9600e8a84] 2024-11-09T00:24:32,151 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a9b9478dd5b14016bc0c728a79b2b64f is 175, key is test_row_0/A:col10/1731111865395/Put/seqid=0 2024-11-09T00:24:32,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742450_1626 (size=22561) 2024-11-09T00:24:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:32,556 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a9b9478dd5b14016bc0c728a79b2b64f 2024-11-09T00:24:32,570 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/db3888ddcf8a4ea59f5291006e9fc5e0 is 50, key is test_row_0/B:col10/1731111865395/Put/seqid=0 2024-11-09T00:24:32,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742451_1627 (size=9857) 2024-11-09T00:24:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:32,975 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/db3888ddcf8a4ea59f5291006e9fc5e0 2024-11-09T00:24:32,989 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/39120bc4b59f42c5818f5226fc29851d is 50, key is test_row_0/C:col10/1731111865395/Put/seqid=0 2024-11-09T00:24:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742452_1628 (size=9857) 2024-11-09T00:24:33,394 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/39120bc4b59f42c5818f5226fc29851d 2024-11-09T00:24:33,402 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/A/a9b9478dd5b14016bc0c728a79b2b64f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a9b9478dd5b14016bc0c728a79b2b64f 2024-11-09T00:24:33,409 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a9b9478dd5b14016bc0c728a79b2b64f, entries=100, sequenceid=379, filesize=22.0 K 2024-11-09T00:24:33,410 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/B/db3888ddcf8a4ea59f5291006e9fc5e0 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/db3888ddcf8a4ea59f5291006e9fc5e0 2024-11-09T00:24:33,414 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/db3888ddcf8a4ea59f5291006e9fc5e0, entries=100, sequenceid=379, filesize=9.6 K 2024-11-09T00:24:33,415 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/.tmp/C/39120bc4b59f42c5818f5226fc29851d as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/39120bc4b59f42c5818f5226fc29851d 2024-11-09T00:24:33,421 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/39120bc4b59f42c5818f5226fc29851d, entries=100, sequenceid=379, filesize=9.6 K 2024-11-09T00:24:33,422 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1692ms, sequenceid=379, compaction requested=true 2024-11-09T00:24:33,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43] to archive 2024-11-09T00:24:33,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:24:33,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bffa341cc1b24ca1a4f2bedbe7ea3f2d 2024-11-09T00:24:33,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/bfb6fd324d13425e946d6af4bedeedc8 2024-11-09T00:24:33,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/1901e88e2954464e90f0fbf802588fcf 2024-11-09T00:24:33,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8fde57b1f689446088c8d688d17a8eee 2024-11-09T00:24:33,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4022cc01ad1a4ae6ae33a5058603d845 2024-11-09T00:24:33,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/7ae7882b8ff54b828c7900cb24adadff 2024-11-09T00:24:33,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/085c969de59b48818fa56ca0c116e680 2024-11-09T00:24:33,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/b3ebfff814d3413294165f464aaea281 2024-11-09T00:24:33,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c857b0e574f44be493115442862a2692 2024-11-09T00:24:33,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/165d9ece6b4c46a4ac59f57204a650cf 2024-11-09T00:24:33,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/5ec50dc25a4d4a4b9fef49891d75db31 2024-11-09T00:24:33,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/28e911d9c09644c7af3451e70f95f549 2024-11-09T00:24:33,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/4d087de5665943f5b9db9a232ceda0f0 2024-11-09T00:24:33,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0b75c145b4b34ee6a45c0f4fd21b2718 2024-11-09T00:24:33,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/fa73734518904fc68d42ce3539ca0a4f 2024-11-09T00:24:33,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/0c54d05138a1486d9ea1a23bef5a8c82 2024-11-09T00:24:33,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/461745e2bae04825888848c9b9ab7b81 2024-11-09T00:24:33,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/ad96964fd47449ceb96c1a765dd51037 2024-11-09T00:24:33,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d48bdd528d5d48668283d4f5b2b7fc92 2024-11-09T00:24:33,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/552e2a5a29d34b5086d92c0c61c2d394 2024-11-09T00:24:33,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a0652c92094240e7b497d644d078e1b5 2024-11-09T00:24:33,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/c7173ea67d1743159145c8d7f7bf552f 2024-11-09T00:24:33,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/61e34adc2da74a1abeed78685a3d13d3 2024-11-09T00:24:33,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/d0b72207d197419daf8de2e815d47a43 2024-11-09T00:24:33,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d7084427eb10467ea9ebbbd35582fab2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9517e86c9e194f8aa3dff9317a576cb8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/43ab21c1d60848b694dd6c1fd36ee653, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/8b476faff4e04b1e8646cbef449f6261, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9f0437bf927643bfa844b2eaaa303809, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/77f51f8df91f4ca29f1106e1b1ef5c25, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0] to archive 2024-11-09T00:24:33,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:24:33,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/bba72e60a60b40ab97988312b73d0fc4 2024-11-09T00:24:33,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/842cdbc570074e219aaea9062ce2c417 2024-11-09T00:24:33,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d7084427eb10467ea9ebbbd35582fab2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d7084427eb10467ea9ebbbd35582fab2 2024-11-09T00:24:33,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/442f27f3bd624f65b7d5ff50ecd18566 2024-11-09T00:24:33,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0d1d503cd3b5419786d8b570f4b25e7a 2024-11-09T00:24:33,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/a24b62127cd74712bf8386ea621ea789 2024-11-09T00:24:33,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/f2f46bb9d3c34ee7ac575c7d4e4d08b3 2024-11-09T00:24:33,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9517e86c9e194f8aa3dff9317a576cb8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9517e86c9e194f8aa3dff9317a576cb8 2024-11-09T00:24:33,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0f366def2e4643e9af124c4a6c200145 2024-11-09T00:24:33,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/309b19db4f124f7d9d0f1f6b0681258a 2024-11-09T00:24:33,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/43ab21c1d60848b694dd6c1fd36ee653 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/43ab21c1d60848b694dd6c1fd36ee653 2024-11-09T00:24:33,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/7fa2a3a5451b4265b2264ed447b832e5 2024-11-09T00:24:33,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/95f4868320a4490091ab52b865d8d0e2 2024-11-09T00:24:33,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/8b476faff4e04b1e8646cbef449f6261 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/8b476faff4e04b1e8646cbef449f6261 2024-11-09T00:24:33,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1d097090014a409c9b5dcc2fbcb6aa1e 2024-11-09T00:24:33,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/450a847a3fb44c55a015d7e52de2f76c 2024-11-09T00:24:33,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9f0437bf927643bfa844b2eaaa303809 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/9f0437bf927643bfa844b2eaaa303809 2024-11-09T00:24:33,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/4aee329715ca4e8eac5c52b8580f0218 2024-11-09T00:24:33,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/1c5008b7523849e1a8600ccd4eaafc5b 2024-11-09T00:24:33,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/77f51f8df91f4ca29f1106e1b1ef5c25 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/77f51f8df91f4ca29f1106e1b1ef5c25 2024-11-09T00:24:33,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/d26b84d79605452990c45a5ea20c1984 2024-11-09T00:24:33,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0be5acc558474c2c8a143e3d67d3d817 2024-11-09T00:24:33,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/407b4cb722934ba4b551d0a195814a8e 2024-11-09T00:24:33,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/87aed9da32d24aa3ad0e9526f08b1be0 2024-11-09T00:24:33,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2a40138eb49b4ea1b23e13e4fa6de74a, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/5f4f0fe56b3d4899b704e616dff23d91, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/957368139b0f41e5ab284dd709256bdd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/23d4bba4296142a7a89ce9bc002240dd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/27b69f9edf0d448b928bdbbaf62f0919, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/fc32b3bb0e60451eb484e3ce5863c14d, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9] to archive 2024-11-09T00:24:33,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T00:24:33,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/20a0557684fb4bc2a5c658ddafdc6860 2024-11-09T00:24:33,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/b2730786000549cdb728bdaa4323943a 2024-11-09T00:24:33,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2a40138eb49b4ea1b23e13e4fa6de74a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2a40138eb49b4ea1b23e13e4fa6de74a 2024-11-09T00:24:33,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b62c7334e1042989f72b1d82de8bff6 2024-11-09T00:24:33,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/51151ca3cb9a48ebbab3806b73c68d03 2024-11-09T00:24:33,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/4b7a21e7625c4bcdbee1e5266681b091 2024-11-09T00:24:33,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/ad2b07566f794137b441ddf02e9379cc 2024-11-09T00:24:33,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/5f4f0fe56b3d4899b704e616dff23d91 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/5f4f0fe56b3d4899b704e616dff23d91 2024-11-09T00:24:33,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/2181e99d1a8648968105f560eebcecc8 2024-11-09T00:24:33,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8490f004db4f4f6ea2d2c134f09d9feb 2024-11-09T00:24:33,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/957368139b0f41e5ab284dd709256bdd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/957368139b0f41e5ab284dd709256bdd 2024-11-09T00:24:33,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/518bd12a777f421891eb4d7af7309841 2024-11-09T00:24:33,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af17c41e88d04eb6899b34cc3a59f6b3 2024-11-09T00:24:33,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/23d4bba4296142a7a89ce9bc002240dd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/23d4bba4296142a7a89ce9bc002240dd 2024-11-09T00:24:33,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/9615e53747e5458f82b2615e2ec780ac 2024-11-09T00:24:33,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/89194ed016624eb9af5e1f862d770248 2024-11-09T00:24:33,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/27b69f9edf0d448b928bdbbaf62f0919 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/27b69f9edf0d448b928bdbbaf62f0919 2024-11-09T00:24:33,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/8ace05ede7c54a678aaf9d8d2fdb4281 2024-11-09T00:24:33,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/af995a0068ff4ed8aeef1e5a83b042d6 2024-11-09T00:24:33,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/fc32b3bb0e60451eb484e3ce5863c14d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/fc32b3bb0e60451eb484e3ce5863c14d 2024-11-09T00:24:33,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/a5c1155eed2a4d5ab8c9d99998f63e6b 2024-11-09T00:24:33,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/90e03d0e92694642b2ea67feced57844 2024-11-09T00:24:33,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/428651884e354950839ddfd5c33022bd 2024-11-09T00:24:33,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/64ff87f269664a4f8cc7189126410dc9 2024-11-09T00:24:33,500 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-09T00:24:33,501 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. 2024-11-09T00:24:33,501 DEBUG [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for c5b55e77595e7e32ac3fc5b9600e8a84: Waiting for close lock at 1731111871729Running coprocessor pre-close hooks at 1731111871729Disabling compacts and flushes for region at 1731111871729Disabling writes for close at 1731111871730 (+1 ms)Obtaining lock to block concurrent updates at 1731111871730Preparing flush snapshotting stores in c5b55e77595e7e32ac3fc5b9600e8a84 at 1731111871730Finished memstore snapshotting TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731111871731 (+1 ms)Flushing stores of TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84. at 1731111871732 (+1 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/A: creating writer at 1731111871733 (+1 ms)Flushing mob file c5b55e77595e7e32ac3fc5b9600e8a84/A: appending metadata at 1731111871739 (+6 ms)Flushing mob file c5b55e77595e7e32ac3fc5b9600e8a84/A: closing flushed file at 1731111871739Flushing c5b55e77595e7e32ac3fc5b9600e8a84/A: appending metadata at 1731111872150 (+411 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/A: closing flushed file at 1731111872151 (+1 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/B: creating writer at 1731111872564 (+413 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/B: appending metadata at 1731111872569 (+5 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/B: closing flushed file at 1731111872569Flushing c5b55e77595e7e32ac3fc5b9600e8a84/C: creating writer at 1731111872984 (+415 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/C: appending metadata at 1731111872988 (+4 ms)Flushing c5b55e77595e7e32ac3fc5b9600e8a84/C: closing flushed file at 1731111872988Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cdeaac8: reopening flushed file at 1731111873401 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a478ec: reopening flushed file at 1731111873409 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4414c48c: reopening flushed file at 1731111873415 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c5b55e77595e7e32ac3fc5b9600e8a84 in 1692ms, sequenceid=379, compaction requested=true at 1731111873422 (+7 ms)Writing region close event to WAL at 1731111873498 (+76 ms)Running coprocessor post-close hooks at 1731111873501 (+3 ms)Closed at 1731111873501 2024-11-09T00:24:33,503 INFO [RS_CLOSE_REGION-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,503 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c5b55e77595e7e32ac3fc5b9600e8a84, regionState=CLOSED 2024-11-09T00:24:33,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 because future has completed 2024-11-09T00:24:33,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T00:24:33,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure c5b55e77595e7e32ac3fc5b9600e8a84, server=ef986b38239e,35861,1731111692607 in 1.9310 sec 2024-11-09T00:24:33,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-09T00:24:33,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c5b55e77595e7e32ac3fc5b9600e8a84, UNASSIGN in 1.9360 sec 2024-11-09T00:24:33,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T00:24:33,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-09T00:24:33,513 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731111873512"}]},"ts":"1731111873512"} 2024-11-09T00:24:33,514 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T00:24:33,514 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T00:24:33,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9510 sec 2024-11-09T00:24:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T00:24:33,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:33,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T00:24:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,717 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T00:24:33,718 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,719 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C, FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits] 2024-11-09T00:24:33,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/725987721a9b400883766b735630f4f7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/725987721a9b400883766b735630f4f7 2024-11-09T00:24:33,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8f553b4fae2e4c6e97582323da4dec1e to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/8f553b4fae2e4c6e97582323da4dec1e 2024-11-09T00:24:33,725 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a9b9478dd5b14016bc0c728a79b2b64f to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/A/a9b9478dd5b14016bc0c728a79b2b64f 2024-11-09T00:24:33,727 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0db080e895da431d86cb44b8ff9d364a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/0db080e895da431d86cb44b8ff9d364a 2024-11-09T00:24:33,727 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/2489e4edb31d41529e40a3f3f04fe3d7 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/2489e4edb31d41529e40a3f3f04fe3d7 2024-11-09T00:24:33,728 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/db3888ddcf8a4ea59f5291006e9fc5e0 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/B/db3888ddcf8a4ea59f5291006e9fc5e0 2024-11-09T00:24:33,730 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/0bc96fa37538448a95383a0e9a936ffc to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/0bc96fa37538448a95383a0e9a936ffc 2024-11-09T00:24:33,731 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/39120bc4b59f42c5818f5226fc29851d to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/39120bc4b59f42c5818f5226fc29851d 2024-11-09T00:24:33,732 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/bee9c75497e747ee9537212ebaaa844a to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/C/bee9c75497e747ee9537212ebaaa844a 2024-11-09T00:24:33,734 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits/382.seqid to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84/recovered.edits/382.seqid 2024-11-09T00:24:33,734 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/default/TestAcidGuarantees/c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,734 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T00:24:33,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:24:33,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T00:24:33,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e35bf2c01634ac0b5ce96ae37135296_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090e35bf2c01634ac0b5ce96ae37135296_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,739 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109108d14a22d6542f59459e0806ecc5056_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109108d14a22d6542f59459e0806ecc5056_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,740 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110915028aa556304e56a409e5814327943d_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110915028aa556304e56a409e5814327943d_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,742 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109310e6b2aa9b84654b61e5bbb45ecd1a6_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109310e6b2aa9b84654b61e5bbb45ecd1a6_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,743 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094013f2718fb04972a9042b20a37d920c_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094013f2718fb04972a9042b20a37d920c_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,744 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094350454cd2454f9b8dc3268267dc0986_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094350454cd2454f9b8dc3268267dc0986_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,746 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094e1f83606ed248f58d834c988db00b26_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094e1f83606ed248f58d834c988db00b26_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,747 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957d043110a9c4ada8869c90f7c44412b_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110957d043110a9c4ada8869c90f7c44412b_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,749 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976b2cb88192e4d84b35d44768e1390b7_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110976b2cb88192e4d84b35d44768e1390b7_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,750 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110994ff71b2606045a1a05582e07d52885f_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110994ff71b2606045a1a05582e07d52885f_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,751 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110999f9c77931534a5586ec6d8328d27c1f_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110999f9c77931534a5586ec6d8328d27c1f_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,753 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aafb92c48b784613946db0ff2fb36dd9_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109aafb92c48b784613946db0ff2fb36dd9_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,754 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b64cbf794dbe4420964f20867a109e87_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b64cbf794dbe4420964f20867a109e87_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,756 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf3db9617b1f4453b6deae02a35de603_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cf3db9617b1f4453b6deae02a35de603_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d8e17a675e064826a29d728c24a852f0_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d8e17a675e064826a29d728c24a852f0_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc7c5140c7324f1baaebca15db89aa99_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dc7c5140c7324f1baaebca15db89aa99_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,761 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0386f1d18324de087776b637b7ca245_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e0386f1d18324de087776b637b7ca245_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6f4ebb34c504c029e805a705afbfe94_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e6f4ebb34c504c029e805a705afbfe94_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed54d3252fb249feb1efb4483c1831e5_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed54d3252fb249feb1efb4483c1831e5_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,766 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f2d287ecd65a4d80b372f519ffe2c727_c5b55e77595e7e32ac3fc5b9600e8a84 to hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f2d287ecd65a4d80b372f519ffe2c727_c5b55e77595e7e32ac3fc5b9600e8a84 2024-11-09T00:24:33,767 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T00:24:33,770 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,774 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T00:24:33,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T00:24:33,778 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,778 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T00:24:33,778 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731111873778"}]},"ts":"9223372036854775807"} 2024-11-09T00:24:33,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T00:24:33,781 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c5b55e77595e7e32ac3fc5b9600e8a84, NAME => 'TestAcidGuarantees,,1731111837608.c5b55e77595e7e32ac3fc5b9600e8a84.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T00:24:33,781 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T00:24:33,781 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731111873781"}]},"ts":"9223372036854775807"} 2024-11-09T00:24:33,783 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T00:24:33,784 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T00:24:33,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-09T00:24:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T00:24:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T00:24:33,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T00:24:33,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236 (was 239), OpenFileDescriptor=443 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=508 (was 559), ProcessCount=11 (was 11), AvailableMemoryMB=3382 (was 4266) 2024-11-09T00:24:33,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T00:24:33,836 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:24:33,836 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:24:33,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:33,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:33,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T00:24:33,836 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T00:24:33,836 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-11-09T00:24:33,837 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef986b38239e,38689,1731111691424 2024-11-09T00:24:33,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T00:24:33,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T00:24:33,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:24:33,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:24:33,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T00:24:33,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T00:24:33,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T00:24:33,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:24:33,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:33,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T00:24:33,882 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef986b38239e,35861,1731111692607' ***** 2024-11-09T00:24:33,882 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T00:24:33,882 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T00:24:33,883 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(959): stopping server ef986b38239e,35861,1731111692607 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef986b38239e:35861. 2024-11-09T00:24:33,883 DEBUG [RS:0;ef986b38239e:35861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T00:24:33,883 DEBUG [RS:0;ef986b38239e:35861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:33,883 INFO [RS:0;ef986b38239e:35861 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T00:24:33,884 INFO [RS:0;ef986b38239e:35861 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T00:24:33,884 INFO [RS:0;ef986b38239e:35861 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T00:24:33,884 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T00:24:33,884 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T00:24:33,884 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T00:24:33,885 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:33,885 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T00:24:33,885 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T00:24:33,885 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T00:24:33,885 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T00:24:33,885 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T00:24:33,885 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T00:24:33,904 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/info/54c00f4d518443c0aba98c2881f32a2e is 95, key is TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6./info:/1731111723214/DeleteFamily/seqid=0 2024-11-09T00:24:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742453_1629 (size=6514) 2024-11-09T00:24:33,918 INFO [regionserver/ef986b38239e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T00:24:34,085 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:34,285 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:34,308 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/info/54c00f4d518443c0aba98c2881f32a2e 2024-11-09T00:24:34,328 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/ns/a5e4d92185b2445db3b639fea15c5c9b is 93, key is TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6./ns:/1731111723167/DeleteFamily/seqid=0 2024-11-09T00:24:34,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742454_1630 (size=5906) 2024-11-09T00:24:34,485 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:34,686 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:34,718 INFO [regionserver/ef986b38239e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T00:24:34,718 INFO [regionserver/ef986b38239e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T00:24:34,732 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/ns/a5e4d92185b2445db3b639fea15c5c9b 2024-11-09T00:24:34,755 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/rep_barrier/9fa14e3329784cecb952198082916452 is 102, key is TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6./rep_barrier:/1731111723167/DeleteFamily/seqid=0 2024-11-09T00:24:34,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742455_1631 (size=6025) 2024-11-09T00:24:34,886 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:35,086 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:35,160 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/rep_barrier/9fa14e3329784cecb952198082916452 2024-11-09T00:24:35,187 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/table/a71d61dee5934eb49207afb5d315d553 is 96, key is TestAcidGuarantees,,1731111697629.9e3137292ff5c555eac85d85b3bc65c6./table:/1731111723167/DeleteFamily/seqid=0 2024-11-09T00:24:35,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742456_1632 (size=5936) 2024-11-09T00:24:35,286 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:35,487 DEBUG [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T00:24:35,591 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/table/a71d61dee5934eb49207afb5d315d553 2024-11-09T00:24:35,594 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/info/54c00f4d518443c0aba98c2881f32a2e as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/info/54c00f4d518443c0aba98c2881f32a2e 2024-11-09T00:24:35,597 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/info/54c00f4d518443c0aba98c2881f32a2e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T00:24:35,598 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/ns/a5e4d92185b2445db3b639fea15c5c9b as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/ns/a5e4d92185b2445db3b639fea15c5c9b 2024-11-09T00:24:35,600 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/ns/a5e4d92185b2445db3b639fea15c5c9b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T00:24:35,600 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/rep_barrier/9fa14e3329784cecb952198082916452 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/rep_barrier/9fa14e3329784cecb952198082916452 2024-11-09T00:24:35,603 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/rep_barrier/9fa14e3329784cecb952198082916452, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T00:24:35,603 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/.tmp/table/a71d61dee5934eb49207afb5d315d553 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/table/a71d61dee5934eb49207afb5d315d553 2024-11-09T00:24:35,606 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/table/a71d61dee5934eb49207afb5d315d553, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T00:24:35,606 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false 2024-11-09T00:24:35,610 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T00:24:35,611 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T00:24:35,611 INFO [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T00:24:35,611 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731111873884Running coprocessor pre-close hooks at 1731111873885 (+1 ms)Disabling compacts and flushes for region at 1731111873885Disabling writes for close at 1731111873885Obtaining lock to block concurrent updates at 1731111873885Preparing flush snapshotting stores in 1588230740 at 1731111873885Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731111873885Flushing stores of hbase:meta,,1.1588230740 at 1731111873886 (+1 ms)Flushing 1588230740/info: creating writer at 1731111873886Flushing 1588230740/info: appending metadata at 1731111873903 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731111873903Flushing 1588230740/ns: creating writer at 1731111874311 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731111874328 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731111874328Flushing 1588230740/rep_barrier: creating writer at 1731111874740 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731111874755 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731111874755Flushing 1588230740/table: creating writer at 1731111875170 (+415 ms)Flushing 1588230740/table: appending metadata at 1731111875187 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731111875187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475df9d6: reopening flushed file at 1731111875594 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55decd93: reopening flushed file at 1731111875597 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6660de6e: reopening flushed file at 1731111875600 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699a44c7: reopening flushed file at 1731111875603 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false at 1731111875606 (+3 ms)Writing region close event to WAL at 1731111875608 (+2 ms)Running coprocessor post-close hooks at 1731111875610 (+2 ms)Closed at 1731111875611 (+1 ms) 2024-11-09T00:24:35,611 DEBUG [RS_CLOSE_META-regionserver/ef986b38239e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T00:24:35,687 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(976): stopping server ef986b38239e,35861,1731111692607; all regions closed. 2024-11-09T00:24:35,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741834_1010 (size=25344) 2024-11-09T00:24:35,698 DEBUG [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/oldWALs 2024-11-09T00:24:35,698 INFO [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef986b38239e%2C35861%2C1731111692607.meta:.meta(num 1731111696961) 2024-11-09T00:24:35,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741833_1009 (size=14260261) 2024-11-09T00:24:35,702 DEBUG [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/oldWALs 2024-11-09T00:24:35,702 INFO [RS:0;ef986b38239e:35861 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef986b38239e%2C35861%2C1731111692607:(num 1731111695895) 2024-11-09T00:24:35,702 DEBUG [RS:0;ef986b38239e:35861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T00:24:35,702 INFO [RS:0;ef986b38239e:35861 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T00:24:35,702 INFO [RS:0;ef986b38239e:35861 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T00:24:35,703 INFO [RS:0;ef986b38239e:35861 {}] hbase.ChoreService(370): Chore service for: regionserver/ef986b38239e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T00:24:35,703 INFO [RS:0;ef986b38239e:35861 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T00:24:35,703 INFO [regionserver/ef986b38239e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T00:24:35,703 INFO [RS:0;ef986b38239e:35861 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35861 2024-11-09T00:24:35,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T00:24:35,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef986b38239e,35861,1731111692607 2024-11-09T00:24:35,803 INFO [RS:0;ef986b38239e:35861 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T00:24:35,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef986b38239e,35861,1731111692607] 2024-11-09T00:24:35,824 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef986b38239e,35861,1731111692607 already deleted, retry=false 2024-11-09T00:24:35,824 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef986b38239e,35861,1731111692607 expired; onlineServers=0 2024-11-09T00:24:35,825 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef986b38239e,38689,1731111691424' ***** 2024-11-09T00:24:35,825 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T00:24:35,825 INFO [M:0;ef986b38239e:38689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T00:24:35,825 INFO [M:0;ef986b38239e:38689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T00:24:35,825 DEBUG [M:0;ef986b38239e:38689 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T00:24:35,825 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T00:24:35,825 DEBUG [M:0;ef986b38239e:38689 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T00:24:35,825 DEBUG [master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.large.0-1731111695472 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.large.0-1731111695472,5,FailOnTimeoutGroup] 2024-11-09T00:24:35,825 DEBUG [master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.small.0-1731111695482 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef986b38239e:0:becomeActiveMaster-HFileCleaner.small.0-1731111695482,5,FailOnTimeoutGroup] 2024-11-09T00:24:35,825 INFO [M:0;ef986b38239e:38689 {}] hbase.ChoreService(370): Chore service for: master/ef986b38239e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T00:24:35,826 INFO [M:0;ef986b38239e:38689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T00:24:35,826 DEBUG [M:0;ef986b38239e:38689 {}] master.HMaster(1795): Stopping service threads 2024-11-09T00:24:35,826 INFO [M:0;ef986b38239e:38689 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T00:24:35,826 INFO [M:0;ef986b38239e:38689 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T00:24:35,826 ERROR [M:0;ef986b38239e:38689 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T00:24:35,827 INFO [M:0;ef986b38239e:38689 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T00:24:35,827 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T00:24:35,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T00:24:35,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T00:24:35,835 DEBUG [M:0;ef986b38239e:38689 {}] zookeeper.ZKUtil(347): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T00:24:35,835 WARN [M:0;ef986b38239e:38689 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T00:24:35,836 INFO [M:0;ef986b38239e:38689 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/.lastflushedseqids 2024-11-09T00:24:35,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742457_1633 (size=72) 2024-11-09T00:24:35,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T00:24:35,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35861-0x1011cf81b9d0001, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T00:24:35,916 INFO [RS:0;ef986b38239e:35861 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T00:24:35,916 INFO [RS:0;ef986b38239e:35861 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef986b38239e,35861,1731111692607; zookeeper connection closed. 2024-11-09T00:24:35,917 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e74bad2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e74bad2 2024-11-09T00:24:35,918 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T00:24:36,249 INFO [M:0;ef986b38239e:38689 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T00:24:36,249 INFO [M:0;ef986b38239e:38689 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T00:24:36,250 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T00:24:36,250 INFO [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:24:36,250 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:24:36,250 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T00:24:36,250 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:24:36,250 INFO [M:0;ef986b38239e:38689 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=744.31 KB heapSize=914.08 KB 2024-11-09T00:24:36,266 DEBUG [M:0;ef986b38239e:38689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20f933dc555c44fd9c2b89e8b64a6e7f is 82, key is hbase:meta,,1/info:regioninfo/1731111697085/Put/seqid=0 2024-11-09T00:24:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742458_1634 (size=5672) 2024-11-09T00:24:36,671 INFO [M:0;ef986b38239e:38689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20f933dc555c44fd9c2b89e8b64a6e7f 2024-11-09T00:24:36,705 DEBUG [M:0;ef986b38239e:38689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af2f44491f5f4d6188cdaee801597df5 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1731111840684/Put/seqid=0 2024-11-09T00:24:36,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742459_1635 (size=42809) 2024-11-09T00:24:36,714 INFO [M:0;ef986b38239e:38689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=743.76 KB at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af2f44491f5f4d6188cdaee801597df5 2024-11-09T00:24:36,716 INFO [M:0;ef986b38239e:38689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for af2f44491f5f4d6188cdaee801597df5 2024-11-09T00:24:36,732 DEBUG [M:0;ef986b38239e:38689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc2c1201815a4a368440606c1e2ad259 is 69, key is ef986b38239e,35861,1731111692607/rs:state/1731111695590/Put/seqid=0 2024-11-09T00:24:36,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073742460_1636 (size=5156) 2024-11-09T00:24:37,136 INFO [M:0;ef986b38239e:38689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2096 (bloomFilter=true), to=hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc2c1201815a4a368440606c1e2ad259 2024-11-09T00:24:37,145 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20f933dc555c44fd9c2b89e8b64a6e7f as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20f933dc555c44fd9c2b89e8b64a6e7f 2024-11-09T00:24:37,148 INFO [M:0;ef986b38239e:38689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20f933dc555c44fd9c2b89e8b64a6e7f, entries=8, sequenceid=2096, filesize=5.5 K 2024-11-09T00:24:37,149 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af2f44491f5f4d6188cdaee801597df5 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/af2f44491f5f4d6188cdaee801597df5 2024-11-09T00:24:37,151 INFO [M:0;ef986b38239e:38689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for af2f44491f5f4d6188cdaee801597df5 2024-11-09T00:24:37,151 INFO [M:0;ef986b38239e:38689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/af2f44491f5f4d6188cdaee801597df5, entries=162, sequenceid=2096, filesize=41.8 K 2024-11-09T00:24:37,152 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc2c1201815a4a368440606c1e2ad259 as hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc2c1201815a4a368440606c1e2ad259 2024-11-09T00:24:37,154 INFO [M:0;ef986b38239e:38689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44111/user/jenkins/test-data/b06ecb6c-fdd8-83bb-9408-e17b00e45a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc2c1201815a4a368440606c1e2ad259, entries=1, sequenceid=2096, filesize=5.0 K 2024-11-09T00:24:37,155 INFO [M:0;ef986b38239e:38689 {}] regionserver.HRegion(3140): Finished flush of dataSize ~744.31 KB/762176, heapSize ~913.78 KB/935712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 905ms, sequenceid=2096, compaction requested=false 2024-11-09T00:24:37,156 INFO [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T00:24:37,156 DEBUG [M:0;ef986b38239e:38689 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731111876250Disabling compacts and flushes for region at 1731111876250Disabling writes for close at 1731111876250Obtaining lock to block concurrent updates at 1731111876250Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731111876250Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=762176, getHeapSize=935952, getOffHeapSize=0, getCellsCount=2283 at 1731111876250Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731111876251 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731111876251Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731111876266 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731111876266Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731111876680 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731111876704 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731111876704Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731111876717 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731111876732 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731111876732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf95f83: reopening flushed file at 1731111877144 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c738c7: reopening flushed file at 1731111877148 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26bca6d5: reopening flushed file at 1731111877151 (+3 ms)Finished flush of dataSize ~744.31 KB/762176, heapSize ~913.78 KB/935712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 905ms, sequenceid=2096, compaction requested=false at 1731111877155 (+4 ms)Writing region close event to WAL at 1731111877156 (+1 ms)Closed at 1731111877156 2024-11-09T00:24:37,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741830_1006 (size=899583) 2024-11-09T00:24:37,159 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T00:24:37,159 INFO [M:0;ef986b38239e:38689 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T00:24:37,159 INFO [M:0;ef986b38239e:38689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38689 2024-11-09T00:24:37,160 INFO [M:0;ef986b38239e:38689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T00:24:37,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T00:24:37,272 INFO [M:0;ef986b38239e:38689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T00:24:37,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38689-0x1011cf81b9d0000, quorum=127.0.0.1:60455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T00:24:37,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T00:24:37,284 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T00:24:37,284 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T00:24:37,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T00:24:37,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.log.dir/,STOPPED} 2024-11-09T00:24:37,288 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T00:24:37,288 WARN [BP-1500709718-172.17.0.3-1731111685778 heartbeating to localhost/127.0.0.1:44111 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T00:24:37,288 WARN [BP-1500709718-172.17.0.3-1731111685778 heartbeating to localhost/127.0.0.1:44111 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1500709718-172.17.0.3-1731111685778 (Datanode Uuid 85de3c15-71d5-4162-8dbc-9224020473f9) service to localhost/127.0.0.1:44111 2024-11-09T00:24:37,288 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T00:24:37,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data1/current/BP-1500709718-172.17.0.3-1731111685778 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T00:24:37,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/cluster_64f92cd0-a49e-0797-1d7a-a6a572786cd0/data/data2/current/BP-1500709718-172.17.0.3-1731111685778 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T00:24:37,291 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T00:24:37,299 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T00:24:37,300 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T00:24:37,300 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T00:24:37,301 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T00:24:37,301 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1d55dd2c-668f-8da0-c218-bcbb52091f68/hadoop.log.dir/,STOPPED} 2024-11-09T00:24:37,316 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T00:24:37,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down